Repository: mars-project/mars

Issue:
Support md.to_numeric
Support pd.to_numeric. pandas docs see https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.to_numeric.html .


Discussion:

Resolving patch:
diff --git a/mars/dataframe/__init__.py b/mars/dataframe/__init__.py
--- a/mars/dataframe/__init__.py
+++ b/mars/dataframe/__init__.py
@@ -20,6 +20,7 @@
 from .base.eval import mars_eval as eval  # pylint: disable=redefined-builtin
 from .base.melt import melt
 from .base.qcut import qcut
+from .base.to_numeric import to_numeric
 from .datasource.from_tensor import dataframe_from_tensor, series_from_tensor
 from .datasource.from_index import series_from_index
 from .datasource.from_records import from_records
diff --git a/mars/dataframe/base/to_numeric.py b/mars/dataframe/base/to_numeric.py
new file mode 100644
--- /dev/null
+++ b/mars/dataframe/base/to_numeric.py
@@ -0,0 +1,210 @@
+# Copyright 1999-2021 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import numpy as np
+import pandas as pd
+from ..operands import DataFrameOperand, DataFrameOperandMixin
+from ...serialization.serializables import StringField
+from ..initializer import Series as asseries
+from ...tensor import tensor as astensor
+from ..core import SERIES_TYPE
+from ...tensor.core import TENSOR_TYPE, TensorOrder
+from ...core import ENTITY_TYPE, OutputType
+
+
+class DataFrameToNumeric(DataFrameOperand, DataFrameOperandMixin):
+
+    errors = StringField("errors")
+    downcast = StringField("downcast")
+
+    def __init__(self, errors="raise", downcast=None, **kw):
+        super().__init__(errors=errors, downcast=downcast, **kw)
+
+    def __call__(self, arg):
+        if isinstance(arg, pd.Series):
+            arg = asseries(arg)
+        elif not isinstance(arg, ENTITY_TYPE):
+            arg = astensor(arg)
+        if arg.ndim != 1:
+            raise ValueError('Input array must be 1 dimensional')
+        if arg.size == 0:
+            raise ValueError('Input array can not be empty')
+
+        if isinstance(arg, asseries):
+            series = arg
+            self.output_types = [OutputType.series]
+            return self.new_series([series], shape=series.shape, name=series.name,
+                                   index_value=series.index_value,
+                                   dtype=series.dtype)
+        else:
+            tensor = arg
+            self.output_types = [OutputType.tensor]
+            dtype = tensor.dtype
+            if dtype.kind == "U":
+                dtype = np.dtype(np.object_)
+            return self.new_tileables([tensor], shape=tensor.shape, dtype=dtype)[0]
+
+    @classmethod
+    def tile(cls, op):
+        in_df = op.inputs[0]
+        out_df = op.outputs[0]
+
+        out_chunks = []
+        for in_chunk in in_df.chunks:
+            out_op = op.copy().reset_key()
+            chunk_kws = []
+            if isinstance(out_df, SERIES_TYPE):
+                chunk_kws.append({
+                    "dtype": out_df.dtype,
+                    "shape": in_chunk.shape,
+                    "index": in_chunk.index,
+                    "index_value": in_chunk.index_value,
+                    "name": in_chunk.name
+                })
+            elif isinstance(out_df, TENSOR_TYPE):
+                chunk_kws.append({
+                    'dtype': out_df.dtype,
+                    'shape': in_chunk.shape,
+                    'order': TensorOrder.C_ORDER,
+                    'index': in_chunk.index,
+                })
+            out_chunks.append(out_op.new_chunk([in_chunk], kws=chunk_kws))
+
+        new_op = op.copy()
+        kw = out_df.params
+        kw['nsplits'] = in_df.nsplits
+        kw['chunks'] = out_chunks
+        return new_op.new_tileables(op.inputs, kws=[kw])
+
+    @classmethod
+    def execute(cls, ctx, op):
+        input_data = ctx[op.inputs[0].key]
+        errors_ = op.errors
+        downcast_ = op.downcast
+        ctx[op.outputs[0].key] = pd.to_numeric(input_data, errors=errors_, downcast=downcast_)
+
+
+def to_numeric(arg, errors="raise", downcast=None):
+    """
+        Convert argument to a numeric type.
+
+        The default return dtype is `float64` or `int64`
+        depending on the data supplied. Use the `downcast` parameter
+        to obtain other dtypes.
+
+        Please note that precision loss may occur if really large numbers
+        are passed in. Due to the internal limitations of `ndarray`, if
+        numbers smaller than `-9223372036854775808` (np.iinfo(np.int64).min)
+        or larger than `18446744073709551615` (np.iinfo(np.uint64).max) are
+        passed in, it is very likely they will be converted to float so that
+        they can stored in an `ndarray`. These warnings apply similarly to
+        `Series` since it internally leverages `ndarray`.
+
+        Parameters
+        ----------
+        arg : scalar, list, tuple, 1-d array, or Series
+            Argument to be converted.
+        errors : {'ignore', 'raise', 'coerce'}, default 'raise'
+            - If 'raise', then invalid parsing will raise an exception.
+            - If 'coerce', then invalid parsing will be set as NaN.
+            - If 'ignore', then invalid parsing will return the input.
+        downcast : {'integer', 'signed', 'unsigned', 'float'}, default None
+            If not None, and if the data has been successfully cast to a
+            numerical dtype (or if the data was numeric to begin with),
+            downcast that resulting data to the smallest numerical dtype
+            possible according to the following rules:
+
+            - 'integer' or 'signed': smallest signed int dtype (min.: np.int8)
+            - 'unsigned': smallest unsigned int dtype (min.: np.uint8)
+            - 'float': smallest float dtype (min.: np.float32)
+
+            As this behaviour is separate from the core conversion to
+            numeric values, any errors raised during the downcasting
+            will be surfaced regardless of the value of the 'errors' input.
+
+            In addition, downcasting will only occur if the size
+            of the resulting data's dtype is strictly larger than
+            the dtype it is to be cast to, so if none of the dtypes
+            checked satisfy that specification, no downcasting will be
+            performed on the data.
+
+        Returns
+        -------
+        ret
+            Numeric if parsing succeeded.
+            Return type depends on input.  Series if Series, otherwise Tensor.
+
+        See Also
+        --------
+        DataFrame.astype : Cast argument to a specified dtype.
+        to_datetime : Convert argument to datetime.
+        to_timedelta : Convert argument to timedelta.
+        numpy.ndarray.astype : Cast a numpy array to a specified type.
+        DataFrame.convert_dtypes : Convert dtypes.
+
+        Examples
+        --------
+        Take separate series and convert to numeric, coercing when told to
+
+        >>> s = md.Series(['1.0', '2', -3])
+        >>> md.to_numeric(s).execute()
+        0    1.0
+        1    2.0
+        2   -3.0
+        dtype: float64
+        >>> md.to_numeric(s, downcast='float').execute()
+        0    1.0
+        1    2.0
+        2   -3.0
+        dtype: float32
+        >>> md.to_numeric(s, downcast='signed').execute()
+        0    1
+        1    2
+        2   -3
+        dtype: int8
+        >>> s = md.Series(['apple', '1.0', '2', -3])
+        >>> md.to_numeric(s, errors='ignore').execute()
+        0    apple
+        1      1.0
+        2        2
+        3       -3
+        dtype: object
+        >>> md.to_numeric(s, errors='coerce').execute()
+        0    NaN
+        1    1.0
+        2    2.0
+        3   -3.0
+        dtype: float64
+
+        Downcasting of nullable integer and floating dtypes is supported:
+
+        >>> s = md.Series([1, 2, 3], dtype="int64")
+        >>> md.to_numeric(s, downcast="integer").execute()
+        0    1
+        1    2
+        2    3
+        dtype: int8
+        >>> s = md.Series([1.0, 2.1, 3.0], dtype="float64")
+        >>> md.to_numeric(s, downcast="float").execute()
+        0    1.0
+        1    2.1
+        2    3.0
+        dtype: float32
+        """
+    if errors not in ("ignore", "raise", "coerce"):
+        raise ValueError("invalid error value specified")
+    if downcast not in (None, "integer", "signed", "unsigned", "float"):
+        raise ValueError("invalid downcasting method provided")
+
+    op = DataFrameToNumeric(errors=errors, downcast=downcast)
+    return op(arg)


Test patch:
diff --git a/mars/dataframe/base/tests/test_base.py b/mars/dataframe/base/tests/test_base.py
--- a/mars/dataframe/base/tests/test_base.py
+++ b/mars/dataframe/base/tests/test_base.py
@@ -20,7 +20,7 @@
 from mars.config import options, option_context
 from mars.core import OutputType, tile
 from mars.core.operand import OperandStage
-from mars.dataframe import eval as mars_eval, cut
+from mars.dataframe import eval as mars_eval, cut, to_numeric
 from mars.dataframe.base import to_gpu, to_cpu, astype
 from mars.dataframe.core import DATAFRAME_TYPE, SERIES_TYPE, SERIES_CHUNK_TYPE, \
     INDEX_TYPE, CATEGORICAL_TYPE, CATEGORICAL_CHUNK_TYPE
@@ -701,6 +701,28 @@ def test_cut():
     assert r.dtype == e.dtype
 
 
+def test_to_numeric():
+    raw = pd.DataFrame({"a":[1.0, 2, 3, -3]})
+    df = from_pandas_df(raw, chunk_size=2)
+
+    with pytest.raises(ValueError):
+        _ = to_numeric(df)
+
+    with pytest.raises(ValueError):
+        _ = to_numeric([['1.0', 1]])
+
+    with pytest.raises(ValueError):
+        _ = to_numeric([])
+
+    s = from_pandas_series(pd.Series(['1.0', '2.0', 1, -2]), chunk_size=2)
+    r = tile(to_numeric(s))
+    assert len(r.chunks) == 2
+    assert isinstance(r, SERIES_TYPE)
+
+    r = tile(to_numeric(['1.0', '2.0', 1, -2]))
+    assert isinstance(r, TENSOR_TYPE)
+
+
 def test_astype():
     s = from_pandas_series(pd.Series([1, 2, 1, 2], name='a'), chunk_size=2)
     with pytest.raises(KeyError):
diff --git a/mars/dataframe/base/tests/test_base_execution.py b/mars/dataframe/base/tests/test_base_execution.py
--- a/mars/dataframe/base/tests/test_base_execution.py
+++ b/mars/dataframe/base/tests/test_base_execution.py
@@ -26,6 +26,7 @@
 from mars.config import options, option_context
 from mars.dataframe import eval as mars_eval, cut, qcut
 from mars.dataframe.base import to_gpu, to_cpu
+from mars.dataframe.base.to_numeric import to_numeric
 from mars.dataframe.base.rebalance import DataFrameRebalance
 from mars.dataframe.datasource.dataframe import from_pandas as from_pandas_df
 from mars.dataframe.datasource.series import from_pandas as from_pandas_series
@@ -858,6 +859,49 @@ def test_cut_execution(setup):
         cut(s3, 3).execute()
 
 
+def test_to_numeric_execition(setup):
+    rs = np.random.RandomState(0)
+    s = pd.Series(rs.randint(5, size=100))
+    s[rs.randint(100)] = np.nan
+
+    # test 1 chunk
+    series = from_pandas_series(s)
+
+    r = to_numeric(series)
+    pd.testing.assert_series_equal(r.execute().fetch(),
+                                   pd.to_numeric(s))
+
+    # test multi chunks
+    series = from_pandas_series(s, chunk_size=20)
+
+    r = to_numeric(series)
+    pd.testing.assert_series_equal(r.execute().fetch(),
+                                   pd.to_numeric(s))
+
+    # test object dtype
+    s = pd.Series(['1.0', 2, -3, '2.0'])
+    series = from_pandas_series(s)
+
+    r = to_numeric(series)
+    pd.testing.assert_series_equal(r.execute().fetch(),
+                                   pd.to_numeric(s))
+
+    # test errors and downcast
+    s = pd.Series(['appple', 2, -3, '2.0'])
+    series = from_pandas_series(s)
+
+    r = to_numeric(series, errors='ignore', downcast='signed')
+    pd.testing.assert_series_equal(r.execute().fetch(),
+                                   pd.to_numeric(s, errors='ignore', downcast='signed'))
+
+    # test list data
+    l = ['1.0', 2, -3, '2.0']
+
+    r = to_numeric(l)
+    np.testing.assert_array_equal(r.execute().fetch(),
+                                   pd.to_numeric(l))
+
+
 def test_q_cut_execution(setup):
     rs = np.random.RandomState(0)
     raw = rs.random(15) * 1000