Repository: superduper-io/superduper

Issue:
[MISC] Remove `pydantic` as dependency by refactoring `configs.py`
`pydantic` in `base/config.py` caused errors with `ray` and is overkill for the job we want to do. 
In addition, we have standard dataclasses all across the board elsewhere.
We should replace all configuration classes with standard dataclasses.


Discussion:

Resolving patch:
diff --git a/superduperdb/base/config.py b/superduperdb/base/config.py
--- a/superduperdb/base/config.py
+++ b/superduperdb/base/config.py
@@ -4,17 +4,19 @@
 canot contain any other imports from this project.
 """
 
+import dataclasses as dc
 import json
 import os
 import typing as t
 from enum import Enum
 
-from .jsonable import Factory, JSONable
-
 _CONFIG_IMMUTABLE = True
 
 
-class BaseConfigJSONable(JSONable):
+@dc.dataclass
+class BaseConfigJSONable:
+    _lock: t.ClassVar[bool] = False
+
     def force_set(self, name, value):
         """
         Forcefully setattr of BaseConfigJSONable instance
@@ -22,7 +24,7 @@ def force_set(self, name, value):
         super().__setattr__(name, value)
 
     def __setattr__(self, name, value):
-        if not _CONFIG_IMMUTABLE:
+        if not _CONFIG_IMMUTABLE or self._lock is False:
             super().__setattr__(name, value)
             return
 
@@ -32,6 +34,7 @@ def __setattr__(self, name, value):
         )
 
 
+@dc.dataclass
 class Retry(BaseConfigJSONable):
     """
     Describes how to retry using the `tenacity` library
@@ -48,6 +51,7 @@ class Retry(BaseConfigJSONable):
     wait_multiplier: float = 1.0
 
 
+@dc.dataclass
 class Cluster(BaseConfigJSONable):
     """
     Describes a connection to distributed work via Dask
@@ -118,6 +122,7 @@ class BytesEncoding(str, Enum):
     BASE64 = 'Str'
 
 
+@dc.dataclass
 class Config(BaseConfigJSONable):
     """
     The data class containing all configurable superduperdb values
@@ -140,10 +145,6 @@ class Config(BaseConfigJSONable):
 
     """
 
-    @property
-    def self_hosted_vector_search(self) -> bool:
-        return self.data_backend == self.cluster.vector_search
-
     data_backend: str = 'mongodb://superduper:superduper@localhost:27017/test_db'
 
     lance_home: str = os.path.join('.superduperdb', 'vector_indices')
@@ -151,8 +152,8 @@ def self_hosted_vector_search(self) -> bool:
     artifact_store: t.Optional[str] = None
     metadata_store: t.Optional[str] = None
 
-    cluster: Cluster = Factory(Cluster)
-    retries: Retry = Factory(Retry)
+    cluster: Cluster = dc.field(default_factory=Cluster)
+    retries: Retry = dc.field(default_factory=Retry)
 
     downloads_folder: t.Optional[str] = None
     fold_probability: float = 0.05
@@ -164,14 +165,16 @@ def self_hosted_vector_search(self) -> bool:
 
     bytes_encoding: BytesEncoding = BytesEncoding.BYTES
 
-    class Config(JSONable.Config):
-        protected_namespaces = ()
-
     def __post_init__(self):
         if self.dot_env:
             import dotenv
 
             dotenv.load_dotenv(self.dot_env)
+        self._lock = True
+
+    @property
+    def self_hosted_vector_search(self) -> bool:
+        return self.data_backend == self.cluster.vector_search
 
     @property
     def hybrid_storage(self):
@@ -182,11 +185,14 @@ def comparables(self):
         """
         A dict of `self` excluding some defined attributes.
         """
-        _dict = self.dict()
+        _dict = dc.asdict(self)
         list(map(_dict.pop, ('cluster', 'retries', 'downloads_folder')))
         return _dict
 
-    def match(self, cfg: dict):
+    def dict(self):
+        return dc.asdict(self)
+
+    def match(self, cfg: t.Dict):
         """
         Match the target cfg dict with `self` comparables dict.
         """
diff --git a/superduperdb/base/configs.py b/superduperdb/base/configs.py
--- a/superduperdb/base/configs.py
+++ b/superduperdb/base/configs.py
@@ -1,3 +1,4 @@
+import dataclasses as dc
 import os
 import typing as t
 from dataclasses import dataclass
@@ -72,7 +73,15 @@ def config(self) -> t.Any:
             kwargs = {}
 
         kwargs = config_dicts.combine_configs((parent, kwargs, env))
-        return self.cls(**kwargs)
+
+        def _dataclass_from_dict(cls, d):
+            try:
+                fieldtypes = {f.name: f.type for f in dc.fields(cls)}
+                return cls(**{f: _dataclass_from_dict(fieldtypes[f], d[f]) for f in d})
+            except Exception:
+                return d
+
+        return _dataclass_from_dict(self.cls, kwargs)
 
 
 def build_config(cfg: t.Optional[Config] = None) -> Config:


Test patch:
diff --git a/test/unittest/base/test_config.py b/test/unittest/base/test_config.py
--- a/test/unittest/base/test_config.py
+++ b/test/unittest/base/test_config.py
@@ -3,7 +3,8 @@
 import pydantic
 import pytest
 
-from superduperdb.base.config import Config, Factory, JSONable
+from superduperdb.base.config import Config
+from superduperdb.base.jsonable import Factory, JSONable
 
 from .test_config_dicts import PARENT
 
@@ -15,22 +16,15 @@
   value is not a valid integer (type=type_error.integer)
 """
 NAME_ERROR = """
-1 validation error for Config
-bad_name
-  extra fields not permitted (type=value_error.extra)
-"""
-NAME_ERROR2 = """
-1 validation error for Config
-bad_name
-  Extra inputs are not permitted [type=extra_forbidden, input_value={}, input_type=dict]
+Config.__init__() got an unexpected keyword argument \'bad_name\'
 """
 
 
 def test_unknown_name():
-    with pytest.raises(pydantic.ValidationError) as pr:
+    with pytest.raises(TypeError) as pr:
         Config(bad_name={})
 
-    expected = (NAME_ERROR2 if IS_2 else NAME_ERROR).strip()
+    expected = NAME_ERROR.strip()
     actual = str(pr.value).strip()
     assert actual.startswith(expected)