Repository: kserve/kserve

Issue:
Make default asyncio workers proportional to cpu-limit
/kind feature

Currently a custom transformer spawns threads promotional to the number of cores on the machine/node. This might be due to python (and tornatdo, asyncio) not being container aware. On nodes with high core count, more threads cause throttling when cpu-limit is comparatively low

Making the number of threads spawned proportional to cpu-limit would help with throttling.



Discussion:

Resolving patch:
diff --git a/python/kfserving/kfserving/kfserver.py b/python/kfserving/kfserving/kfserver.py
--- a/python/kfserving/kfserving/kfserver.py
+++ b/python/kfserving/kfserving/kfserver.py
@@ -22,6 +22,9 @@
 import tornado.web
 import tornado.httpserver
 import tornado.log
+import asyncio
+from tornado import concurrent
+from .utils import utils
 
 from kfserving.handlers.http import PredictHandler, ExplainHandler
 from kfserving import KFModel
@@ -40,6 +43,8 @@
                     help='The max buffer size for tornado.')
 parser.add_argument('--workers', default=1, type=int,
                     help='The number of works to fork')
+parser.add_argument('--max_asyncio_workers', default=None, type=int,
+                    help='Max number of asyncio workers to spawn')
 args, _ = parser.parse_known_args()
 
 tornado.log.enable_pretty_logging()
@@ -50,12 +55,14 @@ def __init__(self, http_port: int = args.http_port,
                  grpc_port: int = args.grpc_port,
                  max_buffer_size: int = args.max_buffer_size,
                  workers: int = args.workers,
+                 max_asyncio_workers: int = args.max_asyncio_workers,
                  registered_models: KFModelRepository = KFModelRepository()):
         self.registered_models = registered_models
         self.http_port = http_port
         self.grpc_port = grpc_port
         self.max_buffer_size = max_buffer_size
         self.workers = workers
+        self.max_asyncio_workers = max_asyncio_workers
         self._http_server: Optional[tornado.httpserver.HTTPServer] = None
 
     def create_application(self):
@@ -87,6 +94,14 @@ def create_application(self):
         ])
 
     def start(self, models: List[KFModel], nest_asyncio: bool = False):
+        if self.max_asyncio_workers is None:
+            # formula as suggest in https://bugs.python.org/issue35279
+            self.max_asyncio_workers = min(32, utils.cpu_count()+4)
+
+        logging.info(f"Setting asyncio max_workers as {self.max_asyncio_workers}")
+        asyncio.get_event_loop().set_default_executor(
+            concurrent.futures.ThreadPoolExecutor(max_workers=self.max_asyncio_workers))
+
         for model in models:
             self.register_model(model)
 
diff --git a/python/kfserving/kfserving/utils/utils.py b/python/kfserving/kfserving/utils/utils.py
--- a/python/kfserving/kfserving/utils/utils.py
+++ b/python/kfserving/kfserving/utils/utils.py
@@ -13,6 +13,8 @@
 # limitations under the License.
 
 import os
+import sys
+import psutil
 
 
 def is_running_in_k8s():
@@ -34,3 +36,36 @@ def set_isvc_namespace(inferenceservice):
     isvc_namespace = inferenceservice.metadata.namespace
     namespace = isvc_namespace or get_default_target_namespace()
     return namespace
+
+
+def cpu_count():
+    """Get the available CPU count for this system.
+    Takes the minimum value from the following locations:
+    - Total system cpus available on the host.
+    - CPU Affinity (if set)
+    - Cgroups limit (if set)
+    """
+    count = os.cpu_count()
+
+    # Check CPU affinity if available
+    try:
+        affinity_count = len(psutil.Process().cpu_affinity())
+        if affinity_count > 0:
+            count = min(count, affinity_count)
+    except Exception:
+        pass
+
+    # Check cgroups if available
+    if sys.platform == "linux":
+        try:
+            with open("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us") as f:
+                quota = int(f.read())
+            with open("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_period_us") as f:
+                period = int(f.read())
+            cgroups_count = int(quota / period)
+            if cgroups_count > 0:
+                count = min(count, cgroups_count)
+        except Exception:
+            pass
+
+    return count


Test patch: