Repository: kserve/kserve

Issue:
Update KServe -> Ray dependency to most recent version
/kind feature

I'd like to update to ray 1.8.0 because they are now building M1 Mac arm64 wheels, which mean's I should be able to install/use the KServe library on my Mac now too.

[Relevant ray issue](https://github.com/ray-project/ray/pull/16621)
[Relevant depency](https://github.com/kserve/kserve/blob/master/python/kserve/requirements.txt#L21)


Discussion:

Resolving patch:
diff --git a/python/kserve/kserve/kfmodel.py b/python/kserve/kserve/kfmodel.py
--- a/python/kserve/kserve/kfmodel.py
+++ b/python/kserve/kserve/kfmodel.py
@@ -20,7 +20,6 @@
 from cloudevents.http import CloudEvent
 from http import HTTPStatus
 from enum import Enum
-from ray.serve.utils import ServeRequest
 import grpc
 from tritonclient.grpc import InferResult, service_pb2_grpc
 from tritonclient.grpc.service_pb2 import ModelInferRequest, ModelInferResponse
@@ -132,8 +131,6 @@ async def preprocess(self, request: Dict) -> Union[Dict, ModelInferRequest]:
                                 status_code=HTTPStatus.BAD_REQUEST,
                                 reason="Unrecognized request format: %s" % e
                             )
-        elif isinstance(request, ServeRequest):
-            return await request.body()
         elif isinstance(request, dict):
 
             if "time" in request \
diff --git a/python/kserve/kserve/kfmodel_repository.py b/python/kserve/kserve/kfmodel_repository.py
--- a/python/kserve/kserve/kfmodel_repository.py
+++ b/python/kserve/kserve/kfmodel_repository.py
@@ -11,7 +11,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import List, Optional, Union
+from typing import Dict, Optional, Union
 from kserve import KFModel
 from ray.serve.api import RayServeHandle
 
@@ -34,8 +34,8 @@ def set_models_dir(self, models_dir):  # used for unit tests
     def get_model(self, name: str) -> Optional[Union[KFModel, RayServeHandle]]:
         return self.models.get(name, None)
 
-    def get_models(self) -> List[Union[KFModel, RayServeHandle]]:
-        return list(self.models.values())
+    def get_models(self) -> Dict[str, Union[KFModel, RayServeHandle]]:
+        return self.models
 
     def is_model_ready(self, name: str):
         model = self.get_model(name)
diff --git a/python/kserve/kserve/kfserver.py b/python/kserve/kserve/kfserver.py
--- a/python/kserve/kserve/kfserver.py
+++ b/python/kserve/kserve/kfserver.py
@@ -102,7 +102,7 @@ def start(self, models: Union[List[KFModel], Dict[str, Deployment]], nest_asynci
                     raise RuntimeError("Model type should be KFModel")
         elif isinstance(models, dict):
             if all([isinstance(v, Deployment) for v in models.values()]):
-                serve.start(detached=True, http_host='0.0.0.0', http_port=9071)
+                serve.start(detached=True, http_options={"host": "0.0.0.0", "port": 9071})
                 for key in models:
                     models[key].deploy()
                     handle = models[key].get_handle()
@@ -166,16 +166,17 @@ def get(self, name: str):
                 reason="Model with name %s does not exist." % name
             )
 
-        if not self.models.is_model_ready(name):
-            raise tornado.web.HTTPError(
-                status_code=503,
-                reason="Model with name %s is not ready." % name
-            )
-
-        self.write({
-            "name": model.name,
-            "ready": model.ready
-        })
+        if self.models.is_model_ready(name):
+            self.write({
+                "name": name,
+                "ready": True
+            })
+        else:
+            self.set_status(503)
+            self.write({
+                "name": name,
+                "ready": False
+            })
 
 
 class ListHandler(BaseHandler):
@@ -183,7 +184,7 @@ def initialize(self, models: KFModelRepository):
         self.models = models  # pylint:disable=attribute-defined-outside-init
 
     def get(self):
-        self.write({"models": [ob.name for ob in self.models.get_models()]})
+        self.write({"models": list(self.models.get_models().keys())})
 
 
 class LoadHandler(BaseHandler):


Test patch:
diff --git a/python/kserve/test/test_server.py b/python/kserve/test/test_server.py
--- a/python/kserve/test/test_server.py
+++ b/python/kserve/test/test_server.py
@@ -22,6 +22,8 @@
 from kserve import kfserver
 from kserve.kfmodel_repository import KFModelRepository
 from tornado.httpclient import HTTPClientError
+from ray import serve
+
 
 test_avsc_schema = '''
         {
@@ -69,6 +71,23 @@ async def explain(self, request):
         return {"predictions": request["instances"]}
 
 
+@serve.deployment
+class DummyServeModel(kfmodel.KFModel):
+    def __init__(self, name):
+        super().__init__(name)
+        self.name = name
+        self.ready = False
+
+    def load(self):
+        self.ready = True
+
+    async def predict(self, request):
+        return {"predictions": request["instances"]}
+
+    async def explain(self, request):
+        return {"predictions": request["instances"]}
+
+
 class DummyCEModel(kfmodel.KFModel):
     def __init__(self, name):
         super().__init__(name)
@@ -203,6 +222,52 @@ async def test_unknown_path(self, http_server_client):
         assert err.value.response.body == b'{"error": "invalid path"}'
 
 
+class TestRayServer:
+    @pytest.fixture(scope="class")
+    def app(self):  # pylint: disable=no-self-use
+        serve.start(detached=False, http_options={"host": "0.0.0.0", "port": 9071})
+
+        DummyServeModel.deploy("TestModel")
+        handle = DummyServeModel.get_handle()
+        handle.load.remote()
+
+        server = kfserver.KFServer()
+        server.register_model_handle("TestModel", handle)
+
+        return server.create_application()
+
+    async def test_liveness_handler(self, http_server_client):
+        resp = await http_server_client.fetch('/')
+        assert resp.code == 200
+        assert resp.body == b'{"status": "alive"}'
+
+    async def test_list_handler(self, http_server_client):
+        resp = await http_server_client.fetch('/v1/models')
+        assert resp.code == 200
+        assert resp.body == b'{"models": ["TestModel"]}'
+
+    async def test_health_handler(self, http_server_client):
+        resp = await http_server_client.fetch('/v1/models/TestModel')
+        assert resp.code == 200
+        assert resp.body == b'{"name": "TestModel", "ready": true}'
+
+    async def test_predict(self, http_server_client):
+        resp = await http_server_client.fetch('/v1/models/TestModel:predict',
+                                              method="POST",
+                                              body=b'{"instances":[[1,2]]}')
+        assert resp.code == 200
+        assert resp.body == b'{"predictions": [[1, 2]]}'
+        assert resp.headers['content-type'] == "application/json; charset=UTF-8"
+
+    async def test_explain(self, http_server_client):
+        resp = await http_server_client.fetch('/v1/models/TestModel:explain',
+                                              method="POST",
+                                              body=b'{"instances":[[1,2]]}')
+        assert resp.code == 200
+        assert resp.body == b'{"predictions": [[1, 2]]}'
+        assert resp.headers['content-type'] == "application/json; charset=UTF-8"
+
+
 class TestTFHttpServerLoadAndUnLoad:
     @pytest.fixture(scope="class")
     def app(self):  # pylint: disable=no-self-use