Repository: mlc-ai/mlc-llm
File: python/mlc_llm/serve/server/server_context.py (Python)

Code before:
        _, config_file_path = _get_model_path(hosted_model)
        with open(config_file_path, "r", encoding="utf-8") as file:
            config = json.load(file)
        self._model_configs[hosted_model] = config

    def get_engine(self, model: str) -> Optional[async_engine.AsyncThreadedEngine]:
        """Get the async engine of the requested model."""
        return self._models.get(model, None)

    def get_conv_template(self, model: str) -> Optional[Conversation]:
        """Get the conversation template of the requested model."""
        conv_template = self._conv_templates.get(model, None)
        if conv_template is not None:
            return conv_template.model_copy(deep=True)
        return None

    def get_model_list(self) -> List[str]:
        """Get the list of models on serve."""
        return list(self._models.keys())

    def get_model_config(self, model: str) -> Optional[Dict]:
        """Get the model config path of the requested model."""
        return self._model_configs.get(model, None)


class ServerContextMiddleware:  # pylint: disable=too-few-public-methods
    """Middleware to inject the server context into the request state."""

    def __init__(self, app, server_context):
        self.app = app
        self.server_context = server_context

    async def __call__(self, scope, receive, send):
        # Add server_context to request state
        scope["server_context"] = self.server_context
        await self.app(scope, receive, send)


Review comment (suggestion):
Let's introduce a static method `ServerContext.current()` to get the current scope of server context, as this is what people more often do. 

Here's a reference https://github.com/apache/tvm/blob/main/python/tvm/relax/block_builder.py#L154-L160.

In `__enter__` we can check there is no current server context, or otherwise we throw an exception.

Code after:
        if hosted_model in self._models:
            raise RuntimeError(f"Model {hosted_model} already running.")
        self._models[hosted_model] = engine

        # Get the conversation template.
        if engine.conv_template_name is not None:
            conv_template = ConvTemplateRegistry.get_conv_template(engine.conv_template_name)
            if conv_template is not None:
                self._conv_templates[hosted_model] = conv_template

        _, config_file_path = _get_model_path(hosted_model)
        with open(config_file_path, "r", encoding="utf-8") as file:
            config = json.load(file)
        self._model_configs[hosted_model] = config

    def get_engine(self, model: str) -> Optional[async_engine.AsyncThreadedEngine]:
        """Get the async engine of the requested model."""
        return self._models.get(model, None)

    def get_conv_template(self, model: str) -> Optional[Conversation]:
        """Get the conversation template of the requested model."""
        conv_template = self._conv_templates.get(model, None)
        if conv_template is not None:
            return conv_template.model_copy(deep=True)
        return None

    def get_model_list(self) -> List[str]:
        """Get the list of models on serve."""
        return list(self._models.keys())

    def get_model_config(self, model: str) -> Optional[Dict]:
        """Get the model config path of the requested model."""
        return self._model_configs.get(model, None)