Repository: SeldonIO/MLServer

Issue:
Is there documentation for configuring modelsetting.json?
Is there documentation for configuring modelsetting.json? I would like to configure the metadata of the model, input/output tensor, etc.


Discussion:
Hey @lianxmfor, good catch! 

Unfortunately, we don't have any docs yet for the different settings (both at the server and model levels). We are planning to add a section documenting this on the short-term as part of [MLServer's new docs site](https://mlserver.readthedocs.io/en/latest/index.html). 

In the meantime, although not ideal, it should be relatively straightforward to see the different fields directly in the `ModelSettings` class of the `mlserver/settings.py` file.

https://github.com/SeldonIO/MLServer/blob/494632287c5c6f0e70f0052b4cd49cd71932ecce/mlserver/settings.py#L49-L66

You can also check some of the internal examples, like the one used for our tests: https://github.com/SeldonIO/MLServer/blob/e34c4b9fe2f646e3f890bd49b0dbfb8c8ba36dab/tests/testdata/model-settings.json

I'll keep this issue opened as part of the ongoing documentation efforts. 
Thank you for providing so much information!

Resolving patch:
diff --git a/docs/conf.py b/docs/conf.py
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -33,17 +33,10 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    #  "sphinx.ext.autodoc",
-    # Creates .nojekyll config
-    #  "sphinx.ext.githubpages",
-    # Converts markdown to rst
     "myst_parser",
-    #  "sphinx.ext.napoleon",
-    # automatically generate API docs
-    # see https://github.com/rtfd/readthedocs.org/issues/1139
-    #  "sphinxcontrib.apidoc",
     "sphinx_search.extension",
     "sphinxcontrib.bibtex",
+    "sphinxcontrib.autodoc_pydantic",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -54,52 +47,10 @@
 # This pattern also affects html_static_path and html_extra_path.
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
-# apidoc settings
-apidoc_module_dir = "../mlserver"
-apidoc_output_dir = "api"
-apidoc_excluded_paths = ["**/*test*"]
-apidoc_module_first = True
-apidoc_separate_modules = True
-apidoc_extra_args = ["-d 6"]
-
-# mock imports
-autodoc_mock_imports = [
-    "pandas",
-    "sklearn",
-    "skimage",
-    "requests",
-    "cv2",
-    "bs4",
-    "keras",
-    "seaborn",
-    "PIL",
-    "tensorflow",
-    "spacy",
-    "numpy",
-    "tensorflow_probability",
-    "scipy",
-    "matplotlib",
-    "creme",
-    "cloudpickle",
-    "fbprophet",
-    "dask",
-    "transformers",
-]
-
-
-# Napoleon settings
-napoleon_google_docstring = True
-napoleon_numpy_docstring = True
-napoleon_include_init_with_doc = True
-napoleon_include_private_with_doc = False
-napoleon_include_special_with_doc = True
-napoleon_use_admonition_for_examples = False
-napoleon_use_admonition_for_notes = False
-napoleon_use_admonition_for_references = False
-napoleon_use_ivar = False
-napoleon_use_param = True
-napoleon_use_rtype = False
+# Pydantic settings
+autodoc_pydantic_settings_show_json = False
 
+# Bibtex settings
 bibtex_bibfiles = ["references.bib"]
 bibtex_reference_style = "author_year"
 
diff --git a/mlserver/grpc/server.py b/mlserver/grpc/server.py
--- a/mlserver/grpc/server.py
+++ b/mlserver/grpc/server.py
@@ -9,6 +9,9 @@
 from .dataplane_pb2_grpc import add_GRPCInferenceServiceServicer_to_server
 from .model_repository_pb2_grpc import add_ModelRepositoryServiceServicer_to_server
 
+# Workers used for non-AsyncIO workloads (which aren't any in our case)
+DefaultGrpcWorkers = 5
+
 
 class GRPCServer:
     def __init__(
@@ -28,7 +31,7 @@ def _create_server(self):
         )
 
         self._server = aio.server(
-            ThreadPoolExecutor(max_workers=self._settings.grpc_workers),
+            ThreadPoolExecutor(max_workers=DefaultGrpcWorkers),
             options=self._get_options(),
         )
 
diff --git a/mlserver/settings.py b/mlserver/settings.py
--- a/mlserver/settings.py
+++ b/mlserver/settings.py
@@ -14,37 +14,54 @@ class Config:
 
     debug: bool = True
 
-    # Model repository folder
+    # Model repository settings
+    """Root of the model repository, where we will search for models."""
     model_repository_root: str = "."
+    """Flag to load all available models automatically at startup."""
     load_models_at_startup: bool = True
 
     # Server metadata
+    """Name of the server."""
     server_name: str = "mlserver"
+    """Version of the server."""
     server_version: str = __version__
+    """Server extensions loaded."""
     extensions: List[str] = []
 
     # Server settings
+    """Host where to listen for connections."""
     host: str = "0.0.0.0"
+    """Port where to listen for HTTP / REST connections."""
     http_port: int = 8080
+    """Port where to listen for gRPC connections."""
     grpc_port: int = 8081
-    grpc_workers: int = 10
+    """Maximum length (i.e. size) of gRPC payloads."""
     grpc_max_message_length: Optional[int] = None
 
 
 class ModelParameters(BaseSettings):
     """
     Parameters that apply only to a particular instance of a model.
-    This can include things like model weights.
-    The main difference with respect to ModelSettings is that parameters can
-    change on each instance (e.g. each version) of the model.
+    This can include things like model weights, or arbitrary ``extra``
+    parameters particular to the underlying inference runtime.
+    The main difference with respect to ``ModelSettings`` is that parameters
+    can change on each instance (e.g. each version) of the model.
     """
 
     class Config:
         env_prefix = ENV_PREFIX_MODEL_SETTINGS
 
+    """
+    URI where the model artifacts can be found.
+    This path must be either absolute or relative to where MLServer is running.
+    """
     uri: Optional[str] = None
+    """Version of the model."""
     version: Optional[str] = None
+    """Format of the model (only available on certain runtimes)."""
     format: Optional[str] = None
+    """Arbitrary settings, dependent on the inference runtime
+    implementation."""
     extra: Optional[dict] = {}
 
 
@@ -52,24 +69,39 @@ class ModelSettings(BaseSettings):
     class Config:
         env_prefix = ENV_PREFIX_MODEL_SETTINGS
 
+    """Name of the model."""
     name: str = ""
 
     # Model metadata
+    """Framework used to train and serialise the model (e.g. sklearn)."""
     platform: str = ""
+    """Versions of dependencies used to train the model (e.g.
+    sklearn/0.20.1)."""
     versions: List[str] = []
+    """Metadata about the inputs accepted by the model."""
     inputs: List[MetadataTensor] = []
+    """Metadata about the outputs returned by the model."""
     outputs: List[MetadataTensor] = []
 
     # Parallel settings
+    """When parallel inference is enabled, number of workers to run inference
+    across."""
     parallel_workers: int = 4
 
     # Adaptive Batching settings (disabled by default)
+    """When adaptive batching is enabled, maximum number of requests to group
+    together in a single batch."""
     max_batch_size: int = 0
+    """When adaptive batching is enabled, maximum amount of time (in seconds)
+    to wait for enough requests to build a full batch."""
     max_batch_time: float = 0.0
 
     # Custom model class implementation
+    """*Python path* to the inference runtime to use to serve this model (e.g.
+    ``mlserver_sklearn.SKLearnModel``)."""
     implementation: PyObject = "mlserver.model.MLModel"  # type: ignore
 
     # Model parameters are meant to be set directly by the MLServer runtime.
     # However, it's also possible to override them manually.
+    """Extra parameters for each instance of this model."""
     parameters: Optional[ModelParameters] = None


Test patch:
diff --git a/tests/grpc/conftest.py b/tests/grpc/conftest.py
--- a/tests/grpc/conftest.py
+++ b/tests/grpc/conftest.py
@@ -36,12 +36,6 @@ def model_infer_request() -> pb.ModelInferRequest:
     return _read_testdata_pb(payload_path, pb.ModelInferRequest)
 
 
-@pytest.fixture
-def grpc_settings(settings: Settings) -> Settings:
-    settings.grpc_workers = 1
-    return settings
-
-
 @pytest.fixture
 def grpc_repository_index_request() -> mr_pb.RepositoryIndexRequest:
     return mr_pb.RepositoryIndexRequest(ready=None)
@@ -58,33 +52,29 @@ def grpc_parameters() -> Dict[str, pb.InferParameter]:
 
 @pytest.fixture
 async def inference_service_stub(
-    grpc_server, grpc_settings: Settings
+    grpc_server, settings: Settings
 ) -> AsyncGenerator[GRPCInferenceServiceStub, None]:
-    async with aio.insecure_channel(
-        f"{grpc_settings.host}:{grpc_settings.grpc_port}"
-    ) as channel:
+    async with aio.insecure_channel(f"{settings.host}:{settings.grpc_port}") as channel:
         yield GRPCInferenceServiceStub(channel)
 
 
 @pytest.fixture
 async def model_repository_service_stub(
-    grpc_server, grpc_settings: Settings
+    grpc_server, settings: Settings
 ) -> AsyncGenerator[ModelRepositoryServiceStub, None]:
-    async with aio.insecure_channel(
-        f"{grpc_settings.host}:{grpc_settings.grpc_port}"
-    ) as channel:
+    async with aio.insecure_channel(f"{settings.host}:{settings.grpc_port}") as channel:
         yield ModelRepositoryServiceStub(channel)
 
 
 @pytest.fixture
 async def grpc_server(
-    grpc_settings: Settings,
+    settings: Settings,
     data_plane: DataPlane,
     model_repository_handlers: ModelRepositoryHandlers,
     sum_model: SumModel,
 ):
     server = GRPCServer(
-        grpc_settings,
+        settings,
         data_plane=data_plane,
         model_repository_handlers=model_repository_handlers,
     )