Repository: SeldonIO/MLServer

Issue:
Consider using `tobytes()` instead of `tolist()` in numpy codec for performance improvement
Our use-case is to make multiple requests to MLServer and all of them will have a 640x640 image data in numpy format as input.

Given that `tobytes()` is way faster than `tolist()` does it make sense to support it by default in the numpy codec here: https://github.com/SeldonIO/MLServer/blob/61d9f078d1f886d14083551f33db6a3146f12745/mlserver/codecs/numpy.py#L54

----

Performance comparison of the two methods:

![Screenshot 2021-07-31 at 9 19 32 PM](https://user-images.githubusercontent.com/161805/127745419-63e80509-ce32-44b2-8b2a-b23636c764d6.jpg)



Discussion:
Hey @gautamborad, thanks for that suggestion! That makes a lot of sense. 

I imagine that `tobytes()` returns a bytes array, so it would fit really well with payloads setting `BYTES` as the datatype (which I imagine would be your case?).

Resolving patch:
diff --git a/mlserver/codecs/numpy.py b/mlserver/codecs/numpy.py
--- a/mlserver/codecs/numpy.py
+++ b/mlserver/codecs/numpy.py
@@ -1,5 +1,7 @@
 import numpy as np
 
+from typing import Any
+
 from ..types import RequestInput, ResponseOutput
 
 from .base import InputCodec, register_input_codec, register_request_codec
@@ -18,27 +20,59 @@
     "FP16": "float16",
     "FP32": "float32",
     "FP64": "float64",
-    "BYTES": "byte",
+    "BYTES": "bytes",
 }
 
 _NumpyToDatatype = {value: key for key, value in _DatatypeToNumpy.items()}
 
 # NOTE: numpy has more types than v2 protocol
 _NumpyToDatatype["object"] = "BYTES"
+_NumpyToDatatype["S"] = "BYTES"
+
 
+def _to_dtype(request_input: RequestInput) -> "np.dtype":
+    dtype = _DatatypeToNumpy[request_input.datatype]
+
+    if request_input.datatype == "BYTES":
+        # bytes have variable size, so need to specify as part of type
+        # TODO: Make elem size variable (and not just the last dimension)
+        elem_size = request_input.shape[-1]
+        return np.dtype((dtype, elem_size))
 
-def _to_dtype(datatype: str) -> "np.dtype":
-    dtype = _DatatypeToNumpy[datatype]
     return np.dtype(dtype)
 
 
 def _to_datatype(dtype: np.dtype) -> str:
     as_str = str(dtype)
+
+    if as_str not in _NumpyToDatatype:
+        # If not present, try with kind
+        as_str = getattr(dtype, "kind")
+
     datatype = _NumpyToDatatype[as_str]
 
     return datatype
 
 
+def _to_ndarray(request_input: RequestInput) -> np.ndarray:
+    data = getattr(request_input.data, "__root__", request_input.data)
+    dtype = _to_dtype(request_input)
+
+    if request_input.datatype == "BYTES":
+        return np.frombuffer(data, dtype)
+
+    return np.array(data, dtype)
+
+
+def _encode_data(data: np.ndarray, datatype: str) -> Any:
+    if datatype == "BYTES":
+        # tobytes is way faster than tolist, although it's harder to serialise
+        # and only makes sense for actual bytes inputs (#253)
+        return data.tobytes()
+
+    return data.flatten().tolist()
+
+
 @register_input_codec
 class NumpyCodec(InputCodec):
     """
@@ -49,19 +83,18 @@ class NumpyCodec(InputCodec):
 
     @classmethod
     def encode(cls, name: str, payload: np.ndarray) -> ResponseOutput:
+        datatype = _to_datatype(payload.dtype)
+
         return ResponseOutput(
             name=name,
-            datatype=_to_datatype(payload.dtype),
+            datatype=datatype,
             shape=list(payload.shape),
-            data=payload.flatten().tolist(),
+            data=_encode_data(payload, datatype),
         )
 
     @classmethod
     def decode(cls, request_input: RequestInput) -> np.ndarray:
-        dtype = _to_dtype(request_input.datatype)
-        data = getattr(request_input.data, "__root__", request_input.data)
-
-        model_data = np.array(data, dtype=dtype)
+        model_data = _to_ndarray(request_input)
 
         # TODO: Check if reshape not valid
         return model_data.reshape(request_input.shape)


Test patch:
diff --git a/tests/codecs/test_numpy.py b/tests/codecs/test_numpy.py
--- a/tests/codecs/test_numpy.py
+++ b/tests/codecs/test_numpy.py
@@ -20,6 +20,10 @@
             RequestInput(name="foo", shape=[2], data=[1, 2], datatype="FP32"),
             np.array([1.0, 2.0]),
         ),
+        (
+            RequestInput(name="foo", shape=[2, 1], data=b"\x01\x02", datatype="BYTES"),
+            np.array([[b"\x01"], [b"\x02"]], dtype=bytes),
+        ),
     ],
 )
 def test_numpy_codec(request_input, payload):
@@ -51,9 +55,10 @@ def test_numpy_codec(request_input, payload):
         (np.float32, "FP32"),
         (np.float64, "FP64"),
         (np.byte, "INT8"),
+        (bytes, "BYTES"),
     ],
 )
-def test_numpy_codec_to_datatype(dtype, datatype):
+def test_to_datatype(dtype, datatype):
     dtype = np.dtype(dtype)
 
     obtained_datatype = _to_datatype(dtype)