Repository: SeldonIO/MLServer

Issue:
merging the huggingface and mlserver batch variable
As per https://github.com/SeldonIO/MLServer/pull/740#discussion_r981259626 it would be possible to just merge both huggingface batch variable into the mlserver batch variable for less redundancy in the `model-settings.json` file.


Discussion:

Resolving patch:
diff --git a/runtimes/huggingface/mlserver_huggingface/common.py b/runtimes/huggingface/mlserver_huggingface/common.py
--- a/runtimes/huggingface/mlserver_huggingface/common.py
+++ b/runtimes/huggingface/mlserver_huggingface/common.py
@@ -6,6 +6,7 @@
 import numpy as np
 from pydantic import BaseSettings
 from mlserver.errors import MLServerError
+from mlserver.settings import ModelSettings
 
 from transformers.pipelines import pipeline
 from transformers.pipelines.base import Pipeline
@@ -54,7 +55,6 @@ class Config:
     pretrained_tokenizer: Optional[str] = None
     optimum_model: bool = False
     device: int = -1
-    batch_size: Optional[int] = None
 
     @property
     def task_name(self):
@@ -107,7 +107,9 @@ def parse_parameters_from_env() -> Dict:
     return parsed_parameters
 
 
-def load_pipeline_from_settings(hf_settings: HuggingFaceSettings) -> Pipeline:
+def load_pipeline_from_settings(
+    hf_settings: HuggingFaceSettings, settings: ModelSettings
+) -> Pipeline:
     """
     TODO
     """
@@ -131,16 +133,17 @@ def load_pipeline_from_settings(hf_settings: HuggingFaceSettings) -> Pipeline:
         # https://github.com/huggingface/optimum/issues/191
         device = -1
 
+    batch_size = 1 if settings.max_batch_size == 0 else settings.max_batch_size
     pp = pipeline(
         hf_settings.task_name,
         model=model,
         tokenizer=tokenizer,
         device=device,
-        batch_size=hf_settings.batch_size,
+        batch_size=batch_size,
     )
 
-    # If batch_size > 0 we need to ensure tokens are padded
-    if hf_settings.batch_size:
+    # If max_batch_size > 0 we need to ensure tokens are padded
+    if settings.max_batch_size:
         pp.tokenizer.pad_token_id = [str(pp.model.config.eos_token_id)]  # type: ignore
 
     return pp
diff --git a/runtimes/huggingface/mlserver_huggingface/runtime.py b/runtimes/huggingface/mlserver_huggingface/runtime.py
--- a/runtimes/huggingface/mlserver_huggingface/runtime.py
+++ b/runtimes/huggingface/mlserver_huggingface/runtime.py
@@ -56,12 +56,6 @@ def __init__(self, settings: ModelSettings):
                     ),
                 )
 
-        if settings.max_batch_size != self.hf_settings.batch_size:
-            logger.warning(
-                f"hf batch_size: {self.hf_settings.batch_size} is different "
-                f"from MLServer max_batch_size: {settings.max_batch_size}"
-            )
-
         super().__init__(settings)
 
     async def load(self) -> bool:
@@ -70,11 +64,14 @@ async def load(self) -> bool:
         print(self.hf_settings.task_name)
         print("loading model...")
         await asyncio.get_running_loop().run_in_executor(
-            None, load_pipeline_from_settings, self.hf_settings
+            None,
+            load_pipeline_from_settings,
+            self.hf_settings,
+            self.settings,
         )
         print("(re)loading model...")
         # Now we load the cached model which should not block asyncio
-        self._model = load_pipeline_from_settings(self.hf_settings)
+        self._model = load_pipeline_from_settings(self.hf_settings, self.settings)
         self._merge_metadata()
         print("model has been loaded!")
         self.ready = True


Test patch: