Repository: caikit/caikit (python)

Issue:
Utility for dynamically creating dataobject classes
## Is your feature request related to a problem? Please describe.

As a user of caikit, I want the ability to make `@dataobject` classes on-the-fly without mucking around with the `metaclass`'s `__new__` function (as seen [here](https://github.com/caikit/caikit/blob/main/caikit/runtime/service_generation/data_stream_source.py#L206))

## Describe the solution you'd like

Add a `make_dataobject` function that encapsulates the logic for easy reuse

## Describe alternatives you've considered

Continue to make it hard so that people don't do it!

## Additional context

On the one hand, this is a usecase that we want to reserve as a power-user-only feature. On the other hand, we do this enough internally around service deduction that the syntax for doing this is too error prone to keep using everywhere.


Pull request description:
Make dataobject 252
## Description

Closes #252 

Patch:
diff --git a/caikit/core/data_model/dataobject.py b/caikit/core/data_model/dataobject.py
index efa08924..703076a1 100644
--- a/caikit/core/data_model/dataobject.py
+++ b/caikit/core/data_model/dataobject.py
@@ -19,7 +19,18 @@ model objects inline without manually defining the protobufs representation
 
 # Standard
 from enum import Enum
-from typing import Any, Callable, List, Type, Union, get_args, get_origin
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Type,
+    Union,
+    get_args,
+    get_origin,
+)
 import dataclasses
 
 # Third Party
@@ -233,6 +244,53 @@ def render_dataobject_protos(interfaces_dir: str):
         proto_class.write_proto_file(interfaces_dir)
 
 
+def make_dataobject(
+    *,
+    name: str,
+    annotations: Dict[str, type],
+    bases: Optional[Iterable[type]] = None,
+    attrs: Optional[Dict[str, Any]] = None,
+    proto_name: Optional[str] = None,
+    **kwargs,
+) -> _DataObjectBaseMetaClass:
+    """Factory function for creating net-new dataobject classes
+
+    WARNING: This is a power-user feature that should be used with caution since
+        dynamically generated dataobject classes have portability issues due to
+        the use of global registries.
+
+    Kwargs:
+        name (str): The name of the class to create
+        annotations (Dict[str, type]): The type annotations for the class
+        bases (Optional[Iterable[type]]): Additional base classes beyond
+            DataObjectBase
+        attrs (Optional[Dict[str, Any]]): Additional class attributes beyond
+            __annotations__
+        proto_name (Optional[str]): Alternate name to use for the name of
+            protobuf message
+
+    Returns:
+        dataobject_class (_DataObjectBaseMetaClass): Programmatically created
+            class derived from DataObjectBase with the given name and
+            annotations
+    """
+    bases = tuple([DataObjectBase] + list(bases or []))
+    attrs = {
+        "__annotations__": annotations,
+        **(attrs or {}),
+    }
+    if proto_name is not None:
+        kwargs["name"] = proto_name
+    return dataobject(**kwargs)(
+        _DataObjectBaseMetaClass.__new__(
+            _DataObjectBaseMetaClass,
+            name=name,
+            bases=bases,
+            attrs=attrs,
+        )
+    )
+
+
 ## Implementation Details ######################################################
 
 
diff --git a/caikit/runtime/service_generation/data_stream_source.py b/caikit/runtime/service_generation/data_stream_source.py
index 7678de64..e8d6992e 100644
--- a/caikit/runtime/service_generation/data_stream_source.py
+++ b/caikit/runtime/service_generation/data_stream_source.py
@@ -27,12 +27,7 @@ import alog
 
 # Local
 from caikit.core.data_model.base import DataBase
-from caikit.core.data_model.dataobject import (
-    DataObjectBase,
-    _DataObjectBaseMetaClass,
-    _make_oneof_init,
-    dataobject,
-)
+from caikit.core.data_model.dataobject import _make_oneof_init, make_dataobject
 from caikit.core.data_model.streams.data_stream import DataStream
 from caikit.runtime.types.caikit_runtime_exception import CaikitRuntimeException
 import caikit
@@ -202,81 +197,56 @@ def make_data_stream_source(data_element_type: Type) -> Type[DataBase]:
         # these classes can be referenced within the Union annotation for the
         # outer class itself.
         package = "caikit_data_model.runtime"
-        JsonData = dataobject(package=package, name=f"{cls_name}JsonData")(
-            _DataObjectBaseMetaClass.__new__(
-                _DataObjectBaseMetaClass,
-                name="JsonData",
-                bases=(DataObjectBase,),
-                attrs={
-                    "__annotations__": {"data": List[data_element_type]},
-                    "__qualname__": f"{cls_name}.JsonData",
-                },
-            ),
+        JsonData = make_dataobject(
+            package=package,
+            proto_name=f"{cls_name}JsonData",
+            name="JsonData",
+            attrs={"__qualname__": f"{cls_name}.JsonData"},
+            annotations={"data": List[data_element_type]},
         )
-        File = dataobject(package=package, name=f"{cls_name}File")(
-            _DataObjectBaseMetaClass.__new__(
-                _DataObjectBaseMetaClass,
-                name="File",
-                bases=(DataObjectBase,),
-                attrs={
-                    "__annotations__": {"filename": str},
-                    "__qualname__": f"{cls_name}.File",
-                },
-            ),
+        File = make_dataobject(
+            package=package,
+            proto_name=f"{cls_name}File",
+            name="File",
+            attrs={"__qualname__": f"{cls_name}.File"},
+            annotations={"filename": str},
         )
-        ListOfFiles = dataobject(package=package, name=f"{cls_name}ListOfFiles")(
-            _DataObjectBaseMetaClass.__new__(
-                _DataObjectBaseMetaClass,
-                name="ListOfFiles",
-                bases=(DataObjectBase,),
-                attrs={
-                    "__annotations__": {"files": List[str]},
-                    "__qualname__": f"{cls_name}.ListOfFiles",
-                },
-            ),
+        ListOfFiles = make_dataobject(
+            package=package,
+            proto_name=f"{cls_name}ListOfFiles",
+            name="ListOfFiles",
+            attrs={"__qualname__": f"{cls_name}.ListOfFiles"},
+            annotations={"files": List[str]},
         )
-        Directory = dataobject(package=package, name=f"{cls_name}Directory")(
-            _DataObjectBaseMetaClass.__new__(
-                _DataObjectBaseMetaClass,
-                name="Directory",
-                bases=(DataObjectBase,),
-                attrs={
-                    "__annotations__": {
-                        "dirname": str,
-                        "extension": str,
-                    },
-                    "__qualname__": f"{cls_name}.Directory",
-                },
-            ),
+        Directory = make_dataobject(
+            package=package,
+            proto_name=f"{cls_name}Directory",
+            name="Directory",
+            attrs={"__qualname__": f"{cls_name}.Directory"},
+            annotations={"dirname": str, "extension": str},
         )
 
         # Create the outer class that encapsulates the Union (oneof) or the
         # various types of input sources
-        data_object = dataobject(package=package)(
-            _DataObjectBaseMetaClass.__new__(
-                _DataObjectBaseMetaClass,
-                name=cls_name,
-                bases=(DataObjectBase, DataStreamSourceBase),
-                attrs={
-                    "ELEMENT_TYPE": data_element_type,
-                    JsonData.__name__: JsonData,
-                    File.__name__: File,
-                    ListOfFiles.__name__: ListOfFiles,
-                    Directory.__name__: Directory,
-                    "__annotations__": {
-                        "data_stream": Union[
-                            Annotated[JsonData, OneofField(JsonData.__name__.lower())],
-                            Annotated[File, OneofField(File.__name__.lower())],
-                            Annotated[
-                                ListOfFiles, OneofField(ListOfFiles.__name__.lower())
-                            ],
-                            Annotated[
-                                Directory, OneofField(Directory.__name__.lower())
-                            ],
-                        ],
-                    },
-                },
-            )
+        data_object = make_dataobject(
+            package=package,
+            name=cls_name,
+            bases=(DataStreamSourceBase,),
+            attrs={
+                "ELEMENT_TYPE": data_element_type,
+                JsonData.__name__: JsonData,
+                File.__name__: File,
+                ListOfFiles.__name__: ListOfFiles,
+                Directory.__name__: Directory,
+            },
+            annotations={
+                "data_stream": Union[
+                    Annotated[JsonData, OneofField(JsonData.__name__.lower())],
+                    Annotated[File, OneofField(File.__name__.lower())],
+                    Annotated[ListOfFiles, OneofField(ListOfFiles.__name__.lower())],
+                    Annotated[Directory, OneofField(Directory.__name__.lower())],
+                ],
+            },
         )
 
         # Add this data stream source to the common data model and the module
diff --git a/caikit/runtime/service_generation/rpcs.py b/caikit/runtime/service_generation/rpcs.py
index 125ebc29..68c51fc2 100644
--- a/caikit/runtime/service_generation/rpcs.py
+++ b/caikit/runtime/service_generation/rpcs.py
@@ -35,11 +35,7 @@ from .compatibility_checker import ApiFieldNames
 from .data_stream_source import make_data_stream_source
 from caikit.core import ModuleBase, TaskBase
 from caikit.core.data_model.base import DataBase
-from caikit.core.data_model.dataobject import (
-    DataObjectBase,
-    _DataObjectBaseMetaClass,
-    dataobject,
-)
+from caikit.core.data_model.dataobject import make_dataobject
 from caikit.core.signature_parsing import CaikitMethodSignature, CustomSignature
 from caikit.interfaces.runtime.data_model import ModelPointer, TrainingJob
 
@@ -74,7 +70,6 @@ class CaikitRPCBase(abc.ABC):
             if triple[1] in self.request.default_map
         }
         attrs = copy.copy(self.request.default_map)
-        attrs["__annotations__"] = {**properties, **optional_properties}
 
         if not properties and not optional_properties:
             log.warning(
@@ -83,16 +78,12 @@ class CaikitRPCBase(abc.ABC):
             )
             return None
 
-        decorator = dataobject(package=package_name)
-        cls_ = _DataObjectBaseMetaClass.__new__(
-            _DataObjectBaseMetaClass,
+        return make_dataobject(
+            package=package_name,
             name=self.request.name,
-            bases=(DataObjectBase,),
             attrs=attrs,
+            annotations={**properties, **optional_properties},
         )
-        decorated_cls = decorator(cls_)
-
-        return decorated_cls
 
     def create_rpc_json(self, package_name: str) -> Dict:
         """Return json snippet for the service definition of this RPC"""


Test patch:
diff --git a/tests/core/data_model/test_dataobject.py b/tests/core/data_model/test_dataobject.py
index f3751158..aa8151d0 100644
--- a/tests/core/data_model/test_dataobject.py
+++ b/tests/core/data_model/test_dataobject.py
@@ -42,6 +42,8 @@ from caikit.core.data_model.base import DataBase, _DataBaseMetaClass
 from caikit.core.data_model.data_backends.dict_backend import DictBackend
 from caikit.core.data_model.dataobject import (
     _AUTO_GEN_PROTO_CLASSES,
+    CAIKIT_DATA_MODEL,
+    make_dataobject,
     render_dataobject_protos,
 )
 from caikit.core.data_model.enums import isprotobufenum
@@ -922,3 +924,45 @@ def test_dataobject_function_inheritance(temp_dpool):
 
     d_inst = Derived(1)
     assert d_inst.doit() == 3
+
+
+def test_make_dataobject_no_optionals(temp_dpool):
+    """Test that dataobject classes can be created dynamically without optional
+    values given
+    """
+    data_object = make_dataobject(
+        name="FooBar",
+        annotations={"foo": str, "bar": int},
+    )
+
+    assert data_object.__name__ == "FooBar"
+    assert data_object.__annotations__ == {"foo": str, "bar": int}
+    assert issubclass(data_object, DataObjectBase)
+    assert data_object._proto_class.DESCRIPTOR.name == "FooBar"
+    assert data_object._proto_class.DESCRIPTOR.file.package == CAIKIT_DATA_MODEL
+
+
+def test_make_dataobject_with_optionals(temp_dpool):
+    """Test that dataobject classes can be created dynamically with optional
+    values given
+    """
+
+    class OtherBase:
+        pass
+
+    data_object = make_dataobject(
+        name="FooBar",
+        annotations={"foo": str, "bar": int},
+        bases=(OtherBase,),
+        attrs={"prop": "val"},
+        proto_name="SomeOtherFooBar",
+        package="foo.bar.baz",
+    )
+
+    assert data_object.__name__ == "FooBar"
+    assert data_object.__annotations__ == {"foo": str, "bar": int}
+    assert issubclass(data_object, DataObjectBase)
+    assert issubclass(data_object, OtherBase)
+    assert data_object.prop == "val"
+    assert data_object._proto_class.DESCRIPTOR.name == "SomeOtherFooBar"
+    assert data_object._proto_class.DESCRIPTOR.file.package == "foo.bar.baz"