Repository: SeldonIO/MLServer

Issue:
Support for NaN in np.ndarray codec?
Sending/receiving a `numpy` array containing NaN values to/from an mlserver with the `NumpyCodec` results in a `ValueError: Out of range float values are not JSON compliant` error. Is this a known limitation, and are there any good workarounds that would still allow me to use a pre-packaged server? I understand that it would probably be bad to assume than `NaN` can be mapped to `null` in the JSON, but maybe that could be an option?


Discussion:
Hey @carl-dawson ,

What would be the expected behaviour when receiving a `NaN`? In other words, are there any scenario where it shouldn't raise an error (and it may be useful for the model to receive that `NaN`?
Tree based classifiers (e.g. XGBoost) are often fine with `NaN` in the input. In fact, there are some interesting techniques that can be employed so that a model can learn to minimize loss when data is missing (see ss3.4 in [this paper](https://arxiv.org/pdf/1603.02754v3.pdf))
What I have found so far:

- When an infer request is sent with floats and null-s, the null-s are converted into nan of the floating type(e.g. FP32/FP64)
  - Inside, the infer function of the model decides on how to interpret those nan values. It could handle them or not.
  - If the model produces a result with nan-s,  the JSONEncoder does not know how to interpret and convert them to JSON.
    - A possible solution might be allowing nulls in the corresponding pydantic model
- When an infer request is sent with ints and null-s, the null-s are converted into None. The codec decode_input method fails because it doesn’t know how to cast None to int by using the int() function.
  - A possible solution might be to create an np.array of no type and then have it as input to np.array() with a type. This happens in the specific codec module e.g. _to_ndarray(…) in codecs.numpy.py
    - np.array([[1, 2, np.NaN, 3, 4, 5], [3, 4, 5, 6, 7, 0]], dtype=np.int64) – this fails
    - np.array(np.array([[1, 2, np.NaN, 3, 4, 5], [3, 4, 5, 6, 7, 0]]), dtype=np.int64) – this doesn’t

- As a possible solution to deal with nullable values in both REST and gRPC, we could choose a nullable value - max or min or depending on the context - and add additional parameters in the infer request e.g. {“nullableValue“: “-1“}, then in the specific codec, we could replace the -1 values with NaN.

What I have not investigated: how bytes / string type of data behaves when having nulls.

Resolving patch:
diff --git a/mlserver/codecs/numpy.py b/mlserver/codecs/numpy.py
--- a/mlserver/codecs/numpy.py
+++ b/mlserver/codecs/numpy.py
@@ -92,7 +92,24 @@ def _encode_data(data: np.ndarray, datatype: str) -> list:
             # need to encapsulate it into a list so that it's compatible.
             return [data.tobytes()]
 
-    return data.flatten().tolist()
+    flattened_list = data.flatten().tolist()
+
+    # Replace NaN with null
+    if datatype != "BYTES":
+        # The `isnan` method doesn't work on Numpy arrays with non-numeric
+        # types
+        has_nan = np.isnan(data).any()
+        if has_nan:
+            flattened_list = list(map(convert_nan, flattened_list))
+
+    return flattened_list
+
+
+def convert_nan(val):
+    if np.isnan(val):
+        return None
+
+    return val
 
 
 @register_input_codec
diff --git a/mlserver/codecs/pandas.py b/mlserver/codecs/pandas.py
--- a/mlserver/codecs/pandas.py
+++ b/mlserver/codecs/pandas.py
@@ -4,7 +4,7 @@
 from typing import Optional, Any, List, Tuple
 
 from .base import RequestCodec, register_request_codec
-from .numpy import to_datatype, to_dtype
+from .numpy import to_datatype, to_dtype, convert_nan
 from .string import encode_str, StringCodec
 from .utils import get_decoded_or_raw, InputOrOutput, inject_batch_dimension
 from .lists import ListElement
@@ -35,8 +35,13 @@ def _to_series(input_or_output: InputOrOutput) -> pd.Series:
 def _to_response_output(series: pd.Series, use_bytes: bool = True) -> ResponseOutput:
     datatype = to_datatype(series.dtype)
     data = series.tolist()
-    content_type = None
 
+    # Replace NaN with null
+    has_nan = series.isnull().any()
+    if has_nan:
+        data = list(map(convert_nan, data))
+
+    content_type = None
     if datatype == "BYTES":
         data, content_type = _process_bytes(data, use_bytes)
 


Test patch:
diff --git a/tests/codecs/test_numpy.py b/tests/codecs/test_numpy.py
--- a/tests/codecs/test_numpy.py
+++ b/tests/codecs/test_numpy.py
@@ -78,6 +78,26 @@ def test_can_encode(payload: Any, expected: bool):
                 parameters=Parameters(content_type=NumpyCodec.ContentType),
             ),
         ),
+        (
+            np.array([None, "bar"]),
+            ResponseOutput(
+                name="foo",
+                shape=[2, 1],
+                data=[None, "bar"],
+                datatype="BYTES",
+                parameters=Parameters(content_type=NumpyCodec.ContentType),
+            ),
+        ),
+        (
+            np.array([2.3, 3.4, np.NaN]),
+            ResponseOutput(
+                name="foo",
+                shape=[3, 1],
+                data=[2.3, 3.4, None],
+                datatype="FP64",
+                parameters=Parameters(content_type=NumpyCodec.ContentType),
+            ),
+        ),
     ],
 )
 def test_encode_output(payload: np.ndarray, expected: ResponseOutput):
@@ -110,6 +130,10 @@ def test_encode_output(payload: np.ndarray, expected: ResponseOutput):
             RequestInput(name="foo", shape=[2], data=["foo", "bar"], datatype="BYTES"),
             np.array(["foo", "bar"], dtype=str),
         ),
+        (
+            RequestInput(name="foo", shape=[3], data=[1, 2, None], datatype="FP16"),
+            np.array([1, 2, np.NaN], dtype="float16"),
+        ),
     ],
 )
 def test_decode_input(request_input: RequestInput, expected: np.ndarray):
@@ -127,7 +151,7 @@ def test_decode_input(request_input: RequestInput, expected: np.ndarray):
         RequestInput(name="foo", shape=[2], data=["foo", "bar"], datatype="BYTES"),
     ],
 )
-def test_encode_input(request_input):
+def test_codec_idempotent(request_input: RequestInput):
     decoded = NumpyCodec.decode_input(request_input)
     response_output = NumpyCodec.encode_output(name="foo", payload=decoded)
 
diff --git a/tests/codecs/test_pandas.py b/tests/codecs/test_pandas.py
--- a/tests/codecs/test_pandas.py
+++ b/tests/codecs/test_pandas.py
@@ -70,6 +70,16 @@ def test_can_encode(payload: Any, expected: bool):
                 name="bar", shape=[2, 1], data=[[1, 2, 3], [4, 5, 6]], datatype="BYTES"
             ),
         ),
+        (
+            pd.Series(data=[4, np.NaN, 6], name="bar"),
+            True,
+            ResponseOutput(
+                name="bar",
+                shape=[3, 1],
+                data=[4, None, 6],
+                datatype="FP64",
+            ),
+        ),
     ],
 )
 def test_to_response_output(series, use_bytes, expected):
@@ -134,6 +144,23 @@ def test_to_response_output(series, use_bytes, expected):
                 ],
             ),
         ),
+        (
+            pd.DataFrame(
+                {
+                    "a": [1, np.NaN, 3],
+                }
+            ),
+            False,
+            InferenceResponse(
+                model_name="my-model",
+                parameters=Parameters(content_type=PandasCodec.ContentType),
+                outputs=[
+                    ResponseOutput(
+                        name="a", shape=[3, 1], datatype="FP64", data=[1, None, 3]
+                    )
+                ],
+            ),
+        ),
     ],
 )
 def test_encode_response(dataframe, use_bytes, expected):