Repository: kserve/kserve

Issue:
'AioServer.shutdown' was never awaited for kserve fastapi server
/kind bug

**What steps did you take and what happened:**
Create a simple kserve model:
```
import kserve
from typing import Dict
import argparse
import logging

logger = logging.getLogger(__name__)

class CustomModel(kserve.Model):
    def __init__(self, name: str):
        super().__init__(name)
        self.name = name
        self.load()

    def load(self):
        self.ready = True

    def predict(self,request: Dict, headers: Dict ) -> Dict:
        logger.info(request)
        return request


DEFAULT_MODEL_NAME = "custom-model"

parser = argparse.ArgumentParser(parents=[kserve.model_server.parser])
parser.add_argument('--model_name', default=DEFAULT_MODEL_NAME,
                    help='The name that the model is served under.')

args, _ = parser.parse_known_args()
if __name__ == "__main__":
    model = CustomModel(name=args.model_name)
    kserve.ModelServer().start([model])
```
And when you run the server:
```
RuntimeError: Event loop is closed
sys:1: RuntimeWarning: coroutine 'AioServer.shutdown' was never awaited
```

**Anything else you would like to add:**
What worked for me is adding this line before initializing the model:
```
multiprocessing.set_start_method('fork')
```

The above issue only happens on mac, and does not seem to happen with linux containers, this is probably something to do with how to OS defaults to creating new processes.



Discussion:
Since python 3.8 the default multiprocessing start method changed from fork to spawn for Mac and Windows, we are looking to fix this in the patch release 0.10.1.

Resolving patch:
diff --git a/python/kserve/kserve/model_server.py b/python/kserve/kserve/model_server.py
--- a/python/kserve/kserve/model_server.py
+++ b/python/kserve/kserve/model_server.py
@@ -16,16 +16,19 @@
 import asyncio
 import concurrent.futures
 import logging
+import signal
 import socket
 from distutils.util import strtobool
-from typing import List, Dict, Union
+from typing import List, Dict, Union, Optional
 
 from ray import serve as rayserve
 from ray.serve.api import Deployment, RayServeHandle
 
 from .protocol.grpc.server import GRPCServer
-from .protocol.rest.server import UvicornProcess
+from .protocol.rest.server import UvicornServer
 from .utils import utils
+import multiprocessing
+from multiprocessing import Process
 
 from .model import Model
 from .protocol.dataplane import DataPlane
@@ -133,11 +136,27 @@ async def serve():
             serversocket.bind(('0.0.0.0', self.http_port))
             serversocket.listen(5)
 
-            logging.info(f"starting uvicorn with {self.workers} workers")
-            for _ in range(self.workers):
-                server = UvicornProcess(self.http_port, [serversocket],
-                                        self.dataplane, self.model_repository_extension, self.enable_docs_url)
-                server.start()
+            logging.info(f"Starting uvicorn with {self.workers} workers")
+            loop = asyncio.get_event_loop()
+            for sig in [signal.SIGINT, signal.SIGTERM, signal.SIGQUIT]:
+                loop.add_signal_handler(
+                    sig, lambda s=sig: asyncio.create_task(self.stop(sig=s))
+                )
+            self._rest_server = UvicornServer(self.http_port, [serversocket],
+                                              self.dataplane, self.model_repository_extension, self.enable_docs_url)
+            if self.workers == 1:
+                await self._rest_server.run()
+            else:
+                # Since py38 MacOS/Windows defaults to use spawn for starting multiprocessing.
+                # https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
+                # Spawn does not work with FastAPI/uvicorn in multiprocessing mode, use fork for multiprocessing
+                # https://github.com/tiangolo/fastapi/issues/1586
+                multiprocessing.set_start_method('fork')
+                server = UvicornServer(self.http_port, [serversocket],
+                                       self.dataplane, self.model_repository_extension, self.enable_docs_url)
+                for _ in range(self.workers):
+                    p = Process(target=server.run_sync)
+                    p.start()
 
         async def servers_task():
             servers = [serve()]
@@ -147,6 +166,15 @@ async def servers_task():
 
         asyncio.run(servers_task())
 
+    async def stop(self, sig: Optional[int] = None):
+        logging.info("Stopping the model server")
+        if self._rest_server:
+            logging.info("Stopping the rest server")
+            await self._rest_server.stop()
+        if self._grpc_server:
+            logging.info("Stopping the grpc server")
+            await self._grpc_server.stop(sig)
+
     def register_model_handle(self, name: str, model_handle: RayServeHandle):
         self.registered_models.update_handle(name, model_handle)
         logging.info("Registering model handle: %s", name)
diff --git a/python/kserve/kserve/protocol/rest/server.py b/python/kserve/kserve/protocol/rest/server.py
--- a/python/kserve/kserve/protocol/rest/server.py
+++ b/python/kserve/kserve/protocol/rest/server.py
@@ -13,9 +13,8 @@
 # limitations under the License.
 
 import asyncio
-import multiprocessing
 import socket
-from typing import List
+from typing import List, Optional
 from prometheus_client import REGISTRY, exposition
 
 import pkg_resources
@@ -49,6 +48,11 @@ def timing(self, metric_name, timing, tags):
         logging.info(f"{metric_name} {timing}, {tags}")
 
 
+class _NoSignalUvicornServer(uvicorn.Server):
+    def install_signal_handlers(self) -> None:
+        pass
+
+
 class RESTServer:
     def __init__(self, data_plane: DataPlane, model_repository_extension, enable_docs_url=False):
         self.dataplane = data_plane
@@ -118,14 +122,13 @@ def create_application(self) -> FastAPI:
         )
 
 
-class UvicornProcess(multiprocessing.Process):
-
+class UvicornServer:
     def __init__(self, http_port: int, sockets: List[socket.socket],
                  data_plane: DataPlane, model_repository_extension, enable_docs_url):
         super().__init__()
         self.sockets = sockets
-        self.server = RESTServer(data_plane, model_repository_extension, enable_docs_url)
-        app = self.server.create_application()
+        rest_server = RESTServer(data_plane, model_repository_extension, enable_docs_url)
+        app = rest_server.create_application()
         app.add_middleware(
             TimingMiddleware,
             client=PrintTimings(),
@@ -171,10 +174,15 @@ def __init__(self, http_port: int, sockets: List[socket.socket],
                 },
             }
         )
+        self.server = _NoSignalUvicornServer(config=self.cfg)
 
-    def stop(self):
-        self.terminate()
-
-    def run(self):
+    def run_sync(self):
         server = uvicorn.Server(config=self.cfg)
         asyncio.run(server.serve(sockets=self.sockets))
+
+    async def run(self):
+        await self.server.serve(sockets=self.sockets)
+
+    async def stop(self, sig: Optional[int] = None):
+        if self.server:
+            self.server.handle_exit(sig=sig, frame=None)


Test patch: