Repository: kserve/kserve

Issue:
KServe REST Server doesn't check model readiness before attempting inference.
/kind feature

**Describe the solution you'd like**
As of now, it's possible to perform inference over a `Model` when the model is not in the ready state.
Neither the V1, nor V2 REST endpoints check whether `Model.ready` has been set before attempting inference:
- https://github.com/kserve/kserve/blob/236c1dcff60a4932b4db28c6cec9ce6b1dc470f0/python/kserve/kserve/protocol/rest/v1_endpoints.py#L55
- https://github.com/kserve/kserve/blob/236c1dcff60a4932b4db28c6cec9ce6b1dc470f0/python/kserve/kserve/protocol/rest/v2_endpoints.py#L130

I understand that it's expected that the client checks the separate readiness endpoint before sending inference requests, and that the InferenceService itself should not be marked as ready in Kubernetes until the readiness endpoint returns 200, but I think the inference endpoints should raise `ModelNotReady` themselves when appropriate.

Is there a principled reason for not including this that I'm not aware of? Happy to make a PR to fix this if it's deemed necessary. 

**Anything else you would like to add:**

This caused an issue in our deployed environment, where one client which checked the readiness endpoint was failing on a deployed inference service, but another client which did not check the readiness was returning successful predictions, which caused a lot of confusion until we realised we'd forgotten to set the `.ready` flag on our `Model`.



Discussion:

Resolving patch:
diff --git a/python/alibiexplainer/alibiexplainer/explainer.py b/python/alibiexplainer/alibiexplainer/explainer.py
--- a/python/alibiexplainer/alibiexplainer/explainer.py
+++ b/python/alibiexplainer/alibiexplainer/explainer.py
@@ -64,9 +64,6 @@ def __init__(  # pylint:disable=too-many-arguments
         else:
             raise NotImplementedError
 
-    def load(self) -> bool:
-        pass
-
     def _predict_fn(self, arr: Union[np.ndarray, List]) -> np.ndarray:
         instances = []
         for req_data in arr:
diff --git a/python/kserve/kserve/protocol/rest/v1_endpoints.py b/python/kserve/kserve/protocol/rest/v1_endpoints.py
--- a/python/kserve/kserve/protocol/rest/v1_endpoints.py
+++ b/python/kserve/kserve/protocol/rest/v1_endpoints.py
@@ -64,6 +64,11 @@ async def predict(self, model_name: str, request: Request) -> Union[Response, Di
         Returns:
             Dict|Response: Model inference response.
         """
+        model_ready = self.dataplane.model_ready(model_name)
+
+        if not model_ready:
+            raise ModelNotReady(model_name)
+
         body = await request.body()
         headers = dict(request.headers.items())
         infer_request, req_attributes = self.dataplane.decode(body=body,
@@ -89,6 +94,11 @@ async def explain(self, model_name: str, request: Request) -> Union[Response, Di
         Returns:
             Dict: Explainer output.
         """
+        model_ready = self.dataplane.model_ready(model_name)
+
+        if not model_ready:
+            raise ModelNotReady(model_name)
+
         body = await request.body()
         headers = dict(request.headers.items())
         infer_request, req_attributes = self.dataplane.decode(body=body,
diff --git a/python/kserve/kserve/protocol/rest/v2_endpoints.py b/python/kserve/kserve/protocol/rest/v2_endpoints.py
--- a/python/kserve/kserve/protocol/rest/v2_endpoints.py
+++ b/python/kserve/kserve/protocol/rest/v2_endpoints.py
@@ -123,6 +123,11 @@ async def infer(
         if model_version:
             raise NotImplementedError("Model versioning not supported yet.")
 
+        model_ready = self.dataplane.model_ready(model_name)
+
+        if not model_ready:
+            raise ModelNotReady(model_name)
+
         request_headers = dict(raw_request.headers)
         infer_inputs = [InferInput(name=input.name, shape=input.shape, datatype=input.datatype,
                                    data=input.data,


Test patch:
diff --git a/python/kserve/test/test_server.py b/python/kserve/test/test_server.py
--- a/python/kserve/test/test_server.py
+++ b/python/kserve/test/test_server.py
@@ -351,6 +351,7 @@ class TestTFHttpServerCloudEvent:
     @pytest.fixture(scope="class")
     def app(self):  # pylint: disable=no-self-use
         model = DummyCEModel("TestModel")
+        model.load()
         server = ModelServer()
         server.register_model(model)
         rest_server = RESTServer(server.dataplane, server.model_repository_extension)
@@ -489,6 +490,7 @@ class TestTFHttpServerAvroCloudEvent:
     @pytest.fixture(scope="class")
     def app(self):  # pylint: disable=no-self-use
         model = DummyAvroCEModel("TestModel")
+        model.load()
         server = ModelServer()
         server.register_model(model)
         rest_server = RESTServer(server.dataplane, server.model_repository_extension)
@@ -564,3 +566,41 @@ def test_load_fail(self, http_server_client):
     def test_unload_fail(self, http_server_client):
         resp = http_server_client.post('/v2/repository/models/model/unload', data=b'')
         assert resp.status_code == 404
+
+
+class TestTFHttpServerModelNotReady:
+    @pytest.fixture(scope="class")
+    def app(self):  # pylint: disable=no-self-use
+        model = DummyModel("TestModel")
+        server = ModelServer()
+        server.register_model(model)
+        rest_server = RESTServer(server.dataplane, server.model_repository_extension)
+        return rest_server.create_application()
+
+    @pytest.fixture(scope='class')
+    def http_server_client(self, app):
+        return TestClient(app)
+
+    def test_model_not_ready_v1(self, http_server_client):
+        resp = http_server_client.get('/v1/models/TestModel')
+        assert resp.status_code == 503
+
+    def test_model_not_ready_v2(self, http_server_client):
+        resp = http_server_client.get('/v2/models/TestModel/ready')
+        assert resp.status_code == 503
+
+    def test_predict(self, http_server_client):
+        resp = http_server_client.post('/v1/models/TestModel:predict',
+                                       data=b'{"instances":[[1,2]]}')
+        assert resp.status_code == 503
+
+    def test_infer(self, http_server_client):
+        input_data = b'{"inputs": [{"name": "input-0","shape": [1, 2],"datatype": "INT32","data": [[1,2]]}]}'
+        resp = http_server_client.post('/v2/models/TestModel/infer',
+                                       data=input_data)
+        assert resp.status_code == 503
+
+    def test_explain(self, http_server_client):
+        resp = http_server_client.post('/v1/models/TestModel:explain',
+                                       data=b'{"instances":[[1,2]]}')
+        assert resp.status_code == 503