Repository: microsoft/autogen
Pull request: feat!: Add support for model family specification
Adds support for the model client to advertise the family of model it supports. 

This is a non breaking change but the change, however, for a client to properly advertise its family changes are required.

I'm not sure if I like the `ModelFamily` class or not. The idea was to make it easier to find the families that are supported by AutoGen but make it clear other families can be added by extensions and user code. This is the main reason I opted not to use an Enum. Also, an Enum might make component config etc trickier

Closes #3883 

Problem statement:
Add model-family to context of model client
Model families sometimes have different behavior. To allow an agent to modify its behavior based on a give model we should expose model family.

Open question:
- What is the taxonomy of model families?
- How do we allow a user to define a "new" model family while still trying to reduce the surface area for mistakenly specifying one similar to an existing one?

Hints from issue discussion:

Commit message: Add support for model family specification

Patch to review:
diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py
index 3b92a4a51b03..ee4dfe6e0b90 100644
--- a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py
+++ b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py
@@ -256,7 +256,7 @@ def __init__(
             self._system_messages = [SystemMessage(content=system_message)]
         self._tools: List[Tool] = []
         if tools is not None:
-            if model_client.capabilities["function_calling"] is False:
+            if model_client.model_info["function_calling"] is False:
                 raise ValueError("The model does not support function calling.")
             for tool in tools:
                 if isinstance(tool, Tool):
@@ -277,7 +277,7 @@ def __init__(
         self._handoff_tools: List[Tool] = []
         self._handoffs: Dict[str, HandoffBase] = {}
         if handoffs is not None:
-            if model_client.capabilities["function_calling"] is False:
+            if model_client.model_info["function_calling"] is False:
                 raise ValueError("The model does not support function calling, which is needed for handoffs.")
             for handoff in handoffs:
                 if isinstance(handoff, str):
@@ -325,7 +325,7 @@ async def on_messages_stream(
     ) -> AsyncGenerator[AgentEvent | ChatMessage | Response, None]:
         # Add messages to the model context.
         for msg in messages:
-            if isinstance(msg, MultiModalMessage) and self._model_client.capabilities["vision"] is False:
+            if isinstance(msg, MultiModalMessage) and self._model_client.model_info["vision"] is False:
                 raise ValueError("The model does not support vision.")
             await self._model_context.add_message(UserMessage(content=msg.content, source=msg.source))
 
diff --git a/python/packages/autogen-agentchat/tests/test_assistant_agent.py b/python/packages/autogen-agentchat/tests/test_assistant_agent.py
index 48f51c4712ed..ca079ce407b4 100644
--- a/python/packages/autogen-agentchat/tests/test_assistant_agent.py
+++ b/python/packages/autogen-agentchat/tests/test_assistant_agent.py
@@ -19,6 +19,7 @@
 from autogen_core import Image
 from autogen_core.model_context import BufferedChatCompletionContext
 from autogen_core.models import LLMMessage
+from autogen_core.models._model_client import ModelFamily
 from autogen_core.tools import FunctionTool
 from autogen_ext.models.openai import OpenAIChatCompletionClient
 from openai.resources.chat.completions import AsyncCompletions
@@ -387,11 +388,7 @@ async def test_invalid_model_capabilities() -> None:
     model_client = OpenAIChatCompletionClient(
         model=model,
         api_key="",
-        model_capabilities={
-            "vision": False,
-            "function_calling": False,
-            "json_output": False,
-        },
+        model_info={"vision": False, "function_calling": False, "json_output": False, "family": ModelFamily.UNKNOWN},
     )
 
     with pytest.raises(ValueError):
diff --git a/python/packages/autogen-core/src/autogen_core/models/__init__.py b/python/packages/autogen-core/src/autogen_core/models/__init__.py
index 9b12aa702edd..cf356e4e8e32 100644
--- a/python/packages/autogen-core/src/autogen_core/models/__init__.py
+++ b/python/packages/autogen-core/src/autogen_core/models/__init__.py
@@ -1,4 +1,4 @@
-from ._model_client import ChatCompletionClient, ModelCapabilities
+from ._model_client import ChatCompletionClient, ModelCapabilities, ModelFamily, ModelInfo
 from ._types import (
     AssistantMessage,
     ChatCompletionTokenLogprob,
@@ -27,4 +27,6 @@
     "CreateResult",
     "TopLogprob",
     "ChatCompletionTokenLogprob",
+    "ModelFamily",
+    "ModelInfo",
 ]
diff --git a/python/packages/autogen-core/src/autogen_core/models/_model_client.py b/python/packages/autogen-core/src/autogen_core/models/_model_client.py
index 7a0762a46e91..65210650c256 100644
--- a/python/packages/autogen-core/src/autogen_core/models/_model_client.py
+++ b/python/packages/autogen-core/src/autogen_core/models/_model_client.py
@@ -1,15 +1,10 @@
 from __future__ import annotations
 
+import warnings
 from abc import ABC, abstractmethod
-from typing import Mapping, Optional, Sequence
+from typing import Literal, Mapping, Optional, Sequence, TypeAlias
 
-from typing_extensions import (
-    Any,
-    AsyncGenerator,
-    Required,
-    TypedDict,
-    Union,
-)
+from typing_extensions import Any, AsyncGenerator, Required, TypedDict, Union, deprecated
 
 from .. import CancellationToken
 from .._component_config import ComponentLoader
@@ -17,12 +12,40 @@
 from ._types import CreateResult, LLMMessage, RequestUsage
 
 
+class ModelFamily:
+    """A model family is a group of models that share similar characteristics from a capabilities perspective. This is different to discrete supported features such as vision, function calling, and JSON output.
+
+    This namespace class holds contstants for the model families that AutoGen understands. Other families definitely exist and can be represented by a string, however, AutoGen will treat them as unknown."""
+
+    GPT_4O = "gpt-4o"
+    O1 = "o1"
+    GPT_4 = "gpt-4"
+    GPT_35 = "gpt-35"
+    UNKNOWN = "unknown"
+
+    ANY: TypeAlias = Literal["gpt-4o", "o1", "gpt-4", "gpt-35", "unknown"]
+
+    def __new__(cls, *args: Any, **kwargs: Any) -> None:
+        raise TypeError(f"{cls.__name__} is a namespace class and cannot be instantiated.")
+
+
 class ModelCapabilities(TypedDict, total=False):
     vision: Required[bool]
     function_calling: Required[bool]
     json_output: Required[bool]
 
 
+class ModelInfo(ModelCapabilities, total=False):
+    vision: Required[bool]
+    """True if the model supports vision, aka image input, otherwise False."""
+    function_calling: Required[bool]
+    """True if the model supports function calling, otherwise False."""
+    json_output: Required[bool]
+    """True if the model supports json output, otherwise False. Note: this is different to structured json."""
+    family: Required[ModelFamily.ANY | str]
+    """Model family should be one of the constants from :py:class:`ModelFamily` or a string representing an unknown model family."""
+
+
 class ChatCompletionClient(ABC, ComponentLoader):
     # Caching has to be handled internally as they can depend on the create args that were stored in the constructor
     @abstractmethod
@@ -63,6 +86,20 @@ def count_tokens(self, messages: Sequence[LLMMessage], *, tools: Sequence[Tool |
     @abstractmethod
     def remaining_tokens(self, messages: Sequence[LLMMessage], *, tools: Sequence[Tool | ToolSchema] = []) -> int: ...
 
+    @deprecated(
+        "Use the model_info property instead of the capabilities property. Model info provides more information"
+    )
     @property
     @abstractmethod
     def capabilities(self) -> ModelCapabilities: ...
+
+    @property
+    @abstractmethod
+    def model_info(self) -> ModelInfo:
+        warnings.warn(
+            "Model client in use does not implement model_info property. Falling back to capabilities property. The capabilities property is deprecated and will be removed soon, please implement model_info instead in the model client class.",
+            stacklevel=2,
+        )
+        base_info: ModelInfo = self.capabilities  # type: ignore
+        base_info["family"] = ModelFamily.UNKNOWN
+        return base_info
diff --git a/python/packages/autogen-core/tests/test_tool_agent.py b/python/packages/autogen-core/tests/test_tool_agent.py
index c93815d5dfd9..69a0fc5a862b 100644
--- a/python/packages/autogen-core/tests/test_tool_agent.py
+++ b/python/packages/autogen-core/tests/test_tool_agent.py
@@ -15,6 +15,7 @@
     RequestUsage,
     UserMessage,
 )
+from autogen_core.models._model_client import ModelFamily, ModelInfo
 from autogen_core.tool_agent import (
     InvalidToolArgumentsException,
     ToolAgent,
@@ -141,6 +142,10 @@ def remaining_tokens(self, messages: Sequence[LLMMessage], *, tools: Sequence[To
         def capabilities(self) -> ModelCapabilities:
             return ModelCapabilities(vision=False, function_calling=True, json_output=False)
 
+        @property
+        def model_info(self) -> ModelInfo:
+            return ModelInfo(vision=False, function_calling=True, json_output=False, family=ModelFamily.UNKNOWN)
+
     client = MockChatCompletionClient()
     tools: List[Tool] = [FunctionTool(_pass_function, name="pass", description="Pass function")]
     runtime = SingleThreadedAgentRuntime()
diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py
index 6173a857b74f..3f95cc747730 100644
--- a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py
+++ b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py
@@ -184,11 +184,11 @@ def __init__(
             raise ValueError(
                 "Cannot save screenshots without a debug directory. Set it using the 'debug_dir' parameter. The debug directory is created if it does not exist."
             )
-        if model_client.capabilities["function_calling"] is False:
+        if model_client.model_info["function_calling"] is False:
             raise ValueError(
                 "The model does not support function calling. MultimodalWebSurfer requires a model that supports function calling."
             )
-        if model_client.capabilities["vision"] is False:
+        if model_client.model_info["vision"] is False:
             raise ValueError("The model is not multimodal. MultimodalWebSurfer requires a multimodal model.")
         self._model_client = model_client
         self.headless = headless
diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_model_info.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_model_info.py
index 43ca65ac72d7..d67beb44e1d8 100644
--- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_model_info.py
+++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_model_info.py
@@ -1,6 +1,6 @@
 from typing import Dict
 
-from autogen_core.models import ModelCapabilities
+from autogen_core.models import ModelFamily, ModelInfo
 
 # Based on: https://platform.openai.com/docs/models/continuous-model-upgrades
 # This is a moving target, so correctness is checked by the model value returned by openai against expected values at runtime``
@@ -17,86 +17,102 @@
     "gpt-3.5-turbo-16k": "gpt-3.5-turbo-16k-0613",
 }
 
-_MODEL_CAPABILITIES: Dict[str, ModelCapabilities] = {
+_MODEL_INFO: Dict[str, ModelInfo] = {
     "o1-preview-2024-09-12": {
         "vision": False,
         "function_calling": False,
         "json_output": False,
+        "family": ModelFamily.O1,
     },
     "o1-mini-2024-09-12": {
         "vision": False,
         "function_calling": False,
         "json_output": False,
+        "family": ModelFamily.O1,
     },
     "gpt-4o-2024-08-06": {
         "vision": True,
         "function_calling": True,
         "json_output": True,
+        "family": ModelFamily.GPT_4O,
     },
     "gpt-4o-2024-05-13": {
         "vision": True,
         "function_calling": True,
         "json_output": True,
+        "family": ModelFamily.GPT_4O,
     },
     "gpt-4o-mini-2024-07-18": {
         "vision": True,
         "function_calling": True,
         "json_output": True,
+        "family": ModelFamily.GPT_4O,
     },
     "gpt-4-turbo-2024-04-09": {
         "vision": True,
         "function_calling": True,
         "json_output": True,
+        "family": ModelFamily.GPT_4,
     },
     "gpt-4-0125-preview": {
         "vision": False,
         "function_calling": True,
         "json_output": True,
+        "family": ModelFamily.GPT_4,
     },
     "gpt-4-1106-preview": {
         "vision": False,
         "function_calling": True,
         "json_output": True,
+        "family": ModelFamily.GPT_4,
     },
     "gpt-4-1106-vision-preview": {
         "vision": True,
         "function_calling": False,
         "json_output": False,
+        "family": ModelFamily.GPT_4,
     },
     "gpt-4-0613": {
         "vision": False,
         "function_calling": True,
         "json_output": True,
+        "family": ModelFamily.GPT_4,
     },
     "gpt-4-32k-0613": {
         "vision": False,
         "function_calling": True,
         "json_output": True,
+        "family": ModelFamily.GPT_4,
     },
     "gpt-3.5-turbo-0125": {
         "vision": False,
         "function_calling": True,
         "json_output": True,
+        "family": ModelFamily.GPT_35,
     },
     "gpt-3.5-turbo-1106": {
         "vision": False,
         "function_calling": True,
         "json_output": True,
+        "family": ModelFamily.GPT_35,
     },
     "gpt-3.5-turbo-instruct": {
         "vision": False,
         "function_calling": True,
         "json_output": True,
+        "family": ModelFamily.GPT_35,
     },
     "gpt-3.5-turbo-0613": {
         "vision": False,
         "function_calling": True,
         "json_output": True,
+        "family": ModelFamily.GPT_35,
     },
     "gpt-3.5-turbo-16k-0613": {
         "vision": False,
         "function_calling": True,
         "json_output": True,
+        "family": ModelFamily.GPT_35,
     },
 }
 
@@ -126,9 +142,9 @@ def resolve_model(model: str) -> str:
     return model
 
 
-def get_capabilities(model: str) -> ModelCapabilities:
+def get_info(model: str) -> ModelInfo:
     resolved_model = resolve_model(model)
-    return _MODEL_CAPABILITIES[resolved_model]
+    return _MODEL_INFO[resolved_model]
 
 
 def get_token_limit(model: str) -> int:
diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py
index a804250b3aa1..d7f4b684cd95 100644
--- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py
+++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py
@@ -38,6 +38,8 @@
     FunctionExecutionResultMessage,
     LLMMessage,
     ModelCapabilities,
+    ModelFamily,
+    ModelInfo,
     RequestUsage,
     SystemMessage,
     TopLogprob,
@@ -358,15 +360,23 @@ def __init__(
         *,
         create_args: Dict[str, Any],
         model_capabilities: Optional[ModelCapabilities] = None,
+        model_info: Optional[ModelInfo] = None,
     ):
         self._client = client
-        if model_capabilities is None:
+        if model_capabilities is None and model_info is None:
             try:
-                self._model_capabilities = _model_info.get_capabilities(create_args["model"])
+                self._model_info = _model_info.get_info(create_args["model"])
             except KeyError as err:
                 raise ValueError("model_capabilities is required when model name is not a valid OpenAI model") from err
-        else:
-            self._model_capabilities = model_capabilities
+        elif model_capabilities is not None and model_info is not None:
+            raise ValueError("model_capabilities and model_info are mutually exclusive")
+        elif model_capabilities is not None and model_info is None:
+            warnings.warn("model_capabilities is deprecated, use model_info instead", DeprecationWarning, stacklevel=2)
+            info = cast(ModelInfo, model_capabilities)
+            info["family"] = ModelFamily.UNKNOWN
+            self._model_info = info
+        elif model_capabilities is None and model_info is not None:
+            self._model_info = model_info
 
         self._resolved_model: Optional[str] = None
         if "model" in create_args:
@@ -375,7 +385,7 @@ def __init__(
         if (
             "response_format" in create_args
             and create_args["response_format"]["type"] == "json_object"
-            and not self._model_capabilities["json_output"]
+            and not self._model_info["json_output"]
         ):
             raise ValueError("Model does not support JSON output")
 
@@ -897,7 +907,11 @@ def remaining_tokens(self, messages: Sequence[LLMMessage], *, tools: Sequence[To
 
     @property
     def capabilities(self) -> ModelCapabilities:
-        return self._model_capabilities
+        return self._model_info
+
+    @property
+    def model_info(self) -> ModelInfo:
+        return self._model_info
 
 
 class OpenAIChatCompletionClient(BaseOpenAIChatCompletionClient, Component[OpenAIClientConfigurationConfigModel]):
@@ -973,10 +987,17 @@ def __init__(self, **kwargs: Unpack[OpenAIClientConfiguration]):
             model_capabilities = kwargs["model_capabilities"]
             del copied_args["model_capabilities"]
 
+        model_info: Optional[ModelInfo] = None
+        if "model_info" in kwargs:
+            model_info = kwargs["model_info"]
+            del copied_args["model_info"]
+
         client = _openai_client_from_config(copied_args)
         create_args = _create_args_from_config(copied_args)
         self._raw_config: Dict[str, Any] = copied_args
-        super().__init__(client=client, create_args=create_args, model_capabilities=model_capabilities)
+        super().__init__(
+            client=client, create_args=create_args, model_capabilities=model_capabilities, model_info=model_info
+        )
 
     def __getstate__(self) -> Dict[str, Any]:
         state = self.__dict__.copy()
@@ -1058,10 +1079,17 @@ def __init__(self, **kwargs: Unpack[AzureOpenAIClientConfiguration]):
             model_capabilities = kwargs["model_capabilities"]
             del copied_args["model_capabilities"]
 
+        model_info: Optional[ModelInfo] = None
+        if "model_info" in kwargs:
+            model_info = kwargs["model_info"]
+            del copied_args["model_info"]
+
         client = _azure_openai_client_from_config(copied_args)
         create_args = _create_args_from_config(copied_args)
         self._raw_config: Dict[str, Any] = copied_args
-        super().__init__(client=client, create_args=create_args, model_capabilities=model_capabilities)
+        super().__init__(
+            client=client, create_args=create_args, model_capabilities=model_capabilities, model_info=model_info
+        )
 
     def __getstate__(self) -> Dict[str, Any]:
         state = self.__dict__.copy()
diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/config/__init__.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/config/__init__.py
index c2938cb7c854..d3d63993bd83 100644
--- a/python/packages/autogen-ext/src/autogen_ext/models/openai/config/__init__.py
+++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/config/__init__.py
@@ -1,7 +1,7 @@
 from typing import Awaitable, Callable, Dict, List, Literal, Optional, Union
 
 from autogen_core import ComponentModel
-from autogen_core.models import ModelCapabilities
+from autogen_core.models import ModelCapabilities, ModelInfo
 from pydantic import BaseModel
 from typing_extensions import Required, TypedDict
 
@@ -35,6 +35,7 @@ class BaseOpenAIClientConfiguration(CreateArguments, total=False):
     timeout: Union[float, None]
     max_retries: int
     model_capabilities: ModelCapabilities
+    model_info: ModelInfo
     """What functionality the model supports, determined by default from model name but is overriden if value passed."""
 
 
@@ -84,6 +85,7 @@ class BaseOpenAIClientConfigurationConfigModel(CreateArgumentsConfigModel):
     timeout: float | None = None
     max_retries: int | None = None
     model_capabilities: ModelCapabilities | None = None
+    model_info: ModelInfo | None = None
 
 
 # See OpenAI docs for explanation of these parameters
diff --git a/python/packages/autogen-ext/src/autogen_ext/models/replay/_replay_chat_completion_client.py b/python/packages/autogen-ext/src/autogen_ext/models/replay/_replay_chat_completion_client.py
index 1167be59b8ad..5ab6f167866d 100644
--- a/python/packages/autogen-ext/src/autogen_ext/models/replay/_replay_chat_completion_client.py
+++ b/python/packages/autogen-ext/src/autogen_ext/models/replay/_replay_chat_completion_client.py
@@ -9,6 +9,8 @@
     CreateResult,
     LLMMessage,
     ModelCapabilities,
+    ModelFamily,
+    ModelInfo,
     RequestUsage,
 )
 from autogen_core.tools import Tool, ToolSchema
@@ -119,7 +121,9 @@ def __init__(
     ):
         self.chat_completions = list(chat_completions)
         self.provided_message_count = len(self.chat_completions)
-        self._model_capabilities = ModelCapabilities(vision=False, function_calling=False, json_output=False)
+        self._model_capabilities = ModelInfo(
+            vision=False, function_calling=False, json_output=False, family=ModelFamily.UNKNOWN
+        )
         self._total_available_tokens = 10000
         self._cur_usage = RequestUsage(prompt_tokens=0, completion_tokens=0)
         self._total_usage = RequestUsage(prompt_tokens=0, completion_tokens=0)
@@ -235,6 +239,10 @@ def capabilities(self) -> ModelCapabilities:
         """Return mock capabilities."""
         return self._model_capabilities
 
+    @property
+    def model_info(self) -> ModelInfo:
+        return self._model_capabilities
+
     def reset(self) -> None:
         """Reset the client state and usage to its initial state."""
         self._cur_usage = RequestUsage(prompt_tokens=0, completion_tokens=0)
diff --git a/python/packages/autogen-ext/src/autogen_ext/teams/magentic_one.py b/python/packages/autogen-ext/src/autogen_ext/teams/magentic_one.py
index 94ecc708ace1..65c6ef0ba2e4 100644
--- a/python/packages/autogen-ext/src/autogen_ext/teams/magentic_one.py
+++ b/python/packages/autogen-ext/src/autogen_ext/teams/magentic_one.py
@@ -124,7 +124,7 @@ def __init__(self, client: ChatCompletionClient, hil_mode: bool = False):
         super().__init__(agents, model_client=client)
 
     def _validate_client_capabilities(self, client: ChatCompletionClient) -> None:
-        capabilities = client.capabilities
+        capabilities = client.model_info
         required_capabilities = ["vision", "function_calling", "json_output"]
 
         if not all(capabilities.get(cap) for cap in required_capabilities):
diff --git a/python/packages/autogen-ext/tests/models/test_openai_model_client.py b/python/packages/autogen-ext/tests/models/test_openai_model_client.py
index 54ae8a6ad6e8..73654546e213 100644
--- a/python/packages/autogen-ext/tests/models/test_openai_model_client.py
+++ b/python/packages/autogen-ext/tests/models/test_openai_model_client.py
@@ -14,6 +14,7 @@
     SystemMessage,
     UserMessage,
 )
+from autogen_core.models._model_client import ModelFamily
 from autogen_core.tools import BaseTool, FunctionTool
 from autogen_ext.models.openai import AzureOpenAIChatCompletionClient, OpenAIChatCompletionClient
 from autogen_ext.models.openai._model_info import resolve_model
@@ -139,18 +140,14 @@ async def test_openai_chat_completion_client() -> None:
 
 @pytest.mark.asyncio
 async def test_custom_model_with_capabilities() -> None:
-    with pytest.raises(ValueError, match="model_capabilities is required"):
+    with pytest.raises(ValueError, match="model_info is required"):
         client = OpenAIChatCompletionClient(model="dummy_model", base_url="https://api.dummy.com/v0", api_key="api_key")
 
     client = OpenAIChatCompletionClient(
         model="dummy_model",
         base_url="https://api.dummy.com/v0",
         api_key="api_key",
-        model_capabilities={
-            "vision": False,
-            "function_calling": False,
-            "json_output": False,
-        },
+        model_info={"vision": False, "function_calling": False, "json_output": False, "family": ModelFamily.UNKNOWN},
     )
     assert client
 
@@ -163,7 +160,7 @@ async def test_azure_openai_chat_completion_client() -> None:
         api_key="api_key",
         api_version="2020-08-04",
         azure_endpoint="https://dummy.com",
-        model_capabilities={"vision": True, "function_calling": True, "json_output": True},
+        model_info={"vision": True, "function_calling": True, "json_output": True, "family": ModelFamily.GPT_4O},
     )
     assert client
 


Code review comments:
diff_hunk: @@ -1,28 +1,51 @@
 from __future__ import annotations
 
+import warnings
 from abc import ABC, abstractmethod
-from typing import Mapping, Optional, Sequence
+from typing import Literal, Mapping, Optional, Sequence, TypeAlias
 
-from typing_extensions import (
-    Any,
-    AsyncGenerator,
-    Required,
-    TypedDict,
-    Union,
-)
+from typing_extensions import Any, AsyncGenerator, Required, TypedDict, Union, deprecated
 
 from .. import CancellationToken
 from .._component_config import ComponentLoader
 from ..tools import Tool, ToolSchema
 from ._types import CreateResult, LLMMessage, RequestUsage
 
 
+class ModelFamily:
+    """A model family is a group of models that share similar characteristics from a capabilities perspective. This is different to discrete supported features such as vision, function calling, and JSON output.
+
+    This namespace class holds contstants for the model families that AutoGen understands. Other families definitely exist and can be represented by a string, however, AutoGen will treat them as unknown."""
path: python/packages/autogen-core/src/autogen_core/models/_model_client.py
text: @user1:
The word 'contstants' should be 'constants'.