Repository: caikit/caikit (python)

Issue:
Utility for dynamically creating dataobject classes
## Is your feature request related to a problem? Please describe.

As a user of caikit, I want the ability to make `@dataobject` classes on-the-fly without mucking around with the `metaclass`'s `__new__` function (as seen [here](https://github.com/caikit/caikit/blob/main/caikit/runtime/service_generation/data_stream_source.py#L206))

## Describe the solution you'd like

Add a `make_dataobject` function that encapsulates the logic for easy reuse

## Describe alternatives you've considered

Continue to make it hard so that people don't do it!

## Additional context

On the one hand, this is a usecase that we want to reserve as a power-user-only feature. On the other hand, we do this enough internally around service deduction that the syntax for doing this is too error prone to keep using everywhere.


Pull request:
Make dataobject 252
## Description

Closes #252 

Patch:
diff --git a/caikit/core/data_model/dataobject.py b/caikit/core/data_model/dataobject.py
index efa08924..703076a1 100644
--- a/caikit/core/data_model/dataobject.py
+++ b/caikit/core/data_model/dataobject.py
@@ -19,7 +19,18 @@ model objects inline without manually defining the protobufs representation
 
 # Standard
 from enum import Enum
-from typing import Any, Callable, List, Type, Union, get_args, get_origin
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Type,
+    Union,
+    get_args,
+    get_origin,
+)
 import dataclasses
 
 # Third Party
@@ -233,6 +244,53 @@ def render_dataobject_protos(interfaces_dir: str):
         proto_class.write_proto_file(interfaces_dir)
 
 
+def make_dataobject(
+    *,
+    name: str,
+    annotations: Dict[str, type],
+    bases: Optional[Iterable[type]] = None,
+    attrs: Optional[Dict[str, Any]] = None,
+    proto_name: Optional[str] = None,
+    **kwargs,
+) -> _DataObjectBaseMetaClass:
+    """Factory function for creating net-new dataobject classes
+
+    WARNING: This is a power-user feature that should be used with caution since
+        dynamically generated dataobject classes have portability issues due to
+        the use of global registries.
+
+    Kwargs:
+        name (str): The name of the class to create
+        annotations (Dict[str, type]): The type annotations for the class
+        bases (Optional[Iterable[type]]): Additional base classes beyond
+            DataObjectBase
+        attrs (Optional[Dict[str, Any]]): Additional class attributes beyond
+            __annotations__
+        proto_name (Optional[str]): Alternate name to use for the name of
+            protobuf message
+
+    Returns:
+        dataobject_class (_DataObjectBaseMetaClass): Programmatically created
+            class derived from DataObjectBase with the given name and
+            annotations
+    """
+    bases = tuple([DataObjectBase] + list(bases or []))
+    attrs = {
+        "__annotations__": annotations,
+        **(attrs or {}),
+    }
+    if proto_name is not None:
+        kwargs["name"] = proto_name
+    return dataobject(**kwargs)(
+        _DataObjectBaseMetaClass.__new__(
+            _DataObjectBaseMetaClass,
+            name=name,
+            bases=bases,
+            attrs=attrs,
+        )
+    )
+
+
 ## Implementation Details ######################################################
 
 
diff --git a/caikit/runtime/service_generation/data_stream_source.py b/caikit/runtime/service_generation/data_stream_source.py
index 7678de64..e8d6992e 100644
--- a/caikit/runtime/service_generation/data_stream_source.py
+++ b/caikit/runtime/service_generation/data_stream_source.py
@@ -27,12 +27,7 @@ import alog
 
 # Local
 from caikit.core.data_model.base import DataBase
-from caikit.core.data_model.dataobject import (
-    DataObjectBase,
-    _DataObjectBaseMetaClass,
-    _make_oneof_init,
-    dataobject,
-)
+from caikit.core.data_model.dataobject import _make_oneof_init, make_dataobject
 from caikit.core.data_model.streams.data_stream import DataStream
 from caikit.runtime.types.caikit_runtime_exception import CaikitRuntimeException
 import caikit
@@ -202,81 +197,56 @@ def make_data_stream_source(data_element_type: Type) -> Type[DataBase]:
         # these classes can be referenced within the Union annotation for the
         # outer class itself.
         package = "caikit_data_model.runtime"
-        JsonData = dataobject(package=package, name=f"{cls_name}JsonData")(
-            _DataObjectBaseMetaClass.__new__(
-                _DataObjectBaseMetaClass,
-                name="JsonData",
-                bases=(DataObjectBase,),
-                attrs={
-                    "__annotations__": {"data": List[data_element_type]},
-                    "__qualname__": f"{cls_name}.JsonData",
-                },
-            ),
+        JsonData = make_dataobject(
+            package=package,
+            proto_name=f"{cls_name}JsonData",
+            name="JsonData",
+            attrs={"__qualname__": f"{cls_name}.JsonData"},
+            annotations={"data": List[data_element_type]},
         )
-        File = dataobject(package=package, name=f"{cls_name}File")(
-            _DataObjectBaseMetaClass.__new__(
-                _DataObjectBaseMetaClass,
-                name="File",
-                bases=(DataObjectBase,),
-                attrs={
-                    "__annotations__": {"filename": str},
-                    "__qualname__": f"{cls_name}.File",
-                },
-            ),
+        File = make_dataobject(
+            package=package,
+            proto_name=f"{cls_name}File",
+            name="File",
+            attrs={"__qualname__": f"{cls_name}.File"},
+            annotations={"filename": str},
         )
-        ListOfFiles = dataobject(package=package, name=f"{cls_name}ListOfFiles")(
-            _DataObjectBaseMetaClass.__new__(
-                _DataObjectBaseMetaClass,
-                name="ListOfFiles",
-                bases=(DataObjectBase,),
-                attrs={
-                    "__annotations__": {"files": List[str]},
-                    "__qualname__": f"{cls_name}.ListOfFiles",
-                },
-            ),
+        ListOfFiles = make_dataobject(
+            package=package,
+            proto_name=f"{cls_name}ListOfFiles",
+            name="ListOfFiles",
+            attrs={"__qualname__": f"{cls_name}.ListOfFiles"},
+            annotations={"files": List[str]},
         )
-        Directory = dataobject(package=package, name=f"{cls_name}Directory")(
-            _DataObjectBaseMetaClass.__new__(
-                _DataObjectBaseMetaClass,
-                name="Directory",
-                bases=(DataObjectBase,),
-                attrs={
-                    "__annotations__": {
-                        "dirname": str,
-                        "extension": str,
-                    },
-                    "__qualname__": f"{cls_name}.Directory",
-                },
-            ),
+        Directory = make_dataobject(
+            package=package,
+            proto_name=f"{cls_name}Directory",
+            name="Directory",
+            attrs={"__qualname__": f"{cls_name}.Directory"},
+            annotations={"dirname": str, "extension": str},
         )
 
         # Create the outer class that encapsulates the Union (oneof) or the
         # various types of input sources
-        data_object = dataobject(package=package)(
-            _DataObjectBaseMetaClass.__new__(
-                _DataObjectBaseMetaClass,
-                name=cls_name,
-                bases=(DataObjectBase, DataStreamSourceBase),
-                attrs={
-                    "ELEMENT_TYPE": data_element_type,
-                    JsonData.__name__: JsonData,
-                    File.__name__: File,
-                    ListOfFiles.__name__: ListOfFiles,
-                    Directory.__name__: Directory,
-                    "__annotations__": {
-                        "data_stream": Union[
-                            Annotated[JsonData, OneofField(JsonData.__name__.lower())],
-                            Annotated[File, OneofField(File.__name__.lower())],
-                            Annotated[
-                                ListOfFiles, OneofField(ListOfFiles.__name__.lower())
-                            ],
-                            Annotated[
-                                Directory, OneofField(Directory.__name__.lower())
-                            ],
-                        ],
-                    },
-                },
-            )
+        data_object = make_dataobject(
+            package=package,
+            name=cls_name,
+            bases=(DataStreamSourceBase,),
+            attrs={
+                "ELEMENT_TYPE": data_element_type,
+                JsonData.__name__: JsonData,
+                File.__name__: File,
+                ListOfFiles.__name__: ListOfFiles,
+                Directory.__name__: Directory,
+            },
+            annotations={
+                "data_stream": Union[
+                    Annotated[JsonData, OneofField(JsonData.__name__.lower())],
+                    Annotated[File, OneofField(File.__name__.lower())],
+                    Annotated[ListOfFiles, OneofField(ListOfFiles.__name__.lower())],
+                    Annotated[Directory, OneofField(Directory.__name__.lower())],
+                ],
+            },
         )
 
         # Add this data stream source to the common data model and the module
diff --git a/caikit/runtime/service_generation/rpcs.py b/caikit/runtime/service_generation/rpcs.py
index 125ebc29..68c51fc2 100644
--- a/caikit/runtime/service_generation/rpcs.py
+++ b/caikit/runtime/service_generation/rpcs.py
@@ -35,11 +35,7 @@ from .compatibility_checker import ApiFieldNames
 from .data_stream_source import make_data_stream_source
 from caikit.core import ModuleBase, TaskBase
 from caikit.core.data_model.base import DataBase
-from caikit.core.data_model.dataobject import (
-    DataObjectBase,
-    _DataObjectBaseMetaClass,
-    dataobject,
-)
+from caikit.core.data_model.dataobject import make_dataobject
 from caikit.core.signature_parsing import CaikitMethodSignature, CustomSignature
 from caikit.interfaces.runtime.data_model import ModelPointer, TrainingJob
 
@@ -74,7 +70,6 @@ class CaikitRPCBase(abc.ABC):
             if triple[1] in self.request.default_map
         }
         attrs = copy.copy(self.request.default_map)
-        attrs["__annotations__"] = {**properties, **optional_properties}
 
         if not properties and not optional_properties:
             log.warning(
@@ -83,16 +78,12 @@ class CaikitRPCBase(abc.ABC):
             )
             return None
 
-        decorator = dataobject(package=package_name)
-        cls_ = _DataObjectBaseMetaClass.__new__(
-            _DataObjectBaseMetaClass,
+        return make_dataobject(
+            package=package_name,
             name=self.request.name,
-            bases=(DataObjectBase,),
             attrs=attrs,
+            annotations={**properties, **optional_properties},
         )
-        decorated_cls = decorator(cls_)
-
-        return decorated_cls
 
     def create_rpc_json(self, package_name: str) -> Dict:
         """Return json snippet for the service definition of this RPC"""


Test patch:
diff --git a/tests/core/data_model/test_dataobject.py b/tests/core/data_model/test_dataobject.py
index f3751158..aa8151d0 100644
--- a/tests/core/data_model/test_dataobject.py
+++ b/tests/core/data_model/test_dataobject.py
@@ -42,6 +42,8 @@ from caikit.core.data_model.base import DataBase, _DataBaseMetaClass
 from caikit.core.data_model.data_backends.dict_backend import DictBackend
 from caikit.core.data_model.dataobject import (
     _AUTO_GEN_PROTO_CLASSES,
+    CAIKIT_DATA_MODEL,
+    make_dataobject,
     render_dataobject_protos,
 )
 from caikit.core.data_model.enums import isprotobufenum
@@ -922,3 +924,45 @@ def test_dataobject_function_inheritance(temp_dpool):
 
     d_inst = Derived(1)
     assert d_inst.doit() == 3
+
+
+def test_make_dataobject_no_optionals(temp_dpool):
+    """Test that dataobject classes can be created dynamically without optional
+    values given
+    """
+    data_object = make_dataobject(
+        name="FooBar",
+        annotations={"foo": str, "bar": int},
+    )
+
+    assert data_object.__name__ == "FooBar"
+    assert data_object.__annotations__ == {"foo": str, "bar": int}
+    assert issubclass(data_object, DataObjectBase)
+    assert data_object._proto_class.DESCRIPTOR.name == "FooBar"
+    assert data_object._proto_class.DESCRIPTOR.file.package == CAIKIT_DATA_MODEL
+
+
+def test_make_dataobject_with_optionals(temp_dpool):
+    """Test that dataobject classes can be created dynamically with optional
+    values given
+    """
+
+    class OtherBase:
+        pass
+
+    data_object = make_dataobject(
+        name="FooBar",
+        annotations={"foo": str, "bar": int},
+        bases=(OtherBase,),
+        attrs={"prop": "val"},
+        proto_name="SomeOtherFooBar",
+        package="foo.bar.baz",
+    )
+
+    assert data_object.__name__ == "FooBar"
+    assert data_object.__annotations__ == {"foo": str, "bar": int}
+    assert issubclass(data_object, DataObjectBase)
+    assert issubclass(data_object, OtherBase)
+    assert data_object.prop == "val"
+    assert data_object._proto_class.DESCRIPTOR.name == "SomeOtherFooBar"
+    assert data_object._proto_class.DESCRIPTOR.file.package == "foo.bar.baz"