Repository: delta-io/delta-rs
Pull request #3389: feat: derive macro for config implementations
Author: login: roeap
Labels: name: binding/rust
description: Issues for the Rust crate
Created: 2025-04-16T09:29:03Z | Merged: 2025-04-16T22:50:08Z

# Description

Managing our configuration has become more and more complex over time. Not only are we exposing more and more options, but we also must allow integrations to receive configuration values when we invoke factory methods. This means maintaining different pieces of config in various places and making sure we have consistent behaviours all around. 

In this PR we propose to use derive macros to lessen the burden of maintenance and provide a more robust experience.

When defining a new piece of config we derive `DeltaConfig` and optionally apply additional config to individual fields.

```rs
#[derive(Debug, Clone, Default, DeltaConfig)]
pub struct LimitConfig {
    #[delta(alias = "concurrency_limit", env = "OBJECT_STORE_CONCURRENCY_LIMIT")]
    pub max_concurrency: Option<usize>,
}
```

This will implement the `TryUpdateKey` trait, which is now extended by `TryUpdateKey::load_from_environment` as well as `FromIterator` of string like tuples.

With this PR we would establish a pattern, but there are a few things we may want to consider in the future like adding namespaces `a.b.c` etc ... which would of course only mean updating the macro.

in a follow up PR I would look into fully leveraging the macro throughout all crates.


Diff:
diff --git a/crates/aws/src/storage.rs b/crates/aws/src/storage.rs
index 71a0ee76f..fd463f264 100644
--- a/crates/aws/src/storage.rs
+++ b/crates/aws/src/storage.rs
@@ -13,7 +13,7 @@ use deltalake_core::logstore::object_store::{
     GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, ObjectStoreScheme,
     PutMultipartOpts, PutOptions, PutPayload, PutResult, Result as ObjectStoreResult,
 };
-use deltalake_core::logstore::{str_is_truthy, ObjectStoreFactory, ObjectStoreRef};
+use deltalake_core::logstore::{config::str_is_truthy, ObjectStoreFactory, ObjectStoreRef};
 use deltalake_core::{DeltaResult, DeltaTableError, ObjectStoreError, Path};
 use futures::stream::BoxStream;
 use futures::Future;
diff --git a/crates/catalog-unity/src/lib.rs b/crates/catalog-unity/src/lib.rs
index 25fa53b83..d1295d08b 100644
--- a/crates/catalog-unity/src/lib.rs
+++ b/crates/catalog-unity/src/lib.rs
@@ -35,7 +35,7 @@ use deltalake_core::{
 
 use crate::client::retry::*;
 use deltalake_core::logstore::{
-    object_store_factories, str_is_truthy, IORuntime, ObjectStoreFactory, ObjectStoreRef,
+    config::str_is_truthy, object_store_factories, IORuntime, ObjectStoreFactory, ObjectStoreRef,
 };
 pub mod client;
 pub mod credential;
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index e7374d40f..00352c669 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -15,6 +15,8 @@ rust-version.workspace = true
 features = ["datafusion", "json"]
 
 [dependencies]
+deltalake-derive = { version = "0.25.0", path = "../derive" }
+
 delta_kernel.workspace = true
 
 # arrow
@@ -108,6 +110,7 @@ maplit = "1"
 pretty_assertions = "1.2.1"
 pretty_env_logger = "0.5.0"
 rand = "0.8"
+rstest = { version = "0.25.0" }
 serial_test = "3"
 tempfile = "3"
 tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
diff --git a/crates/core/src/logstore/config.rs b/crates/core/src/logstore/config.rs
index 0e10043c3..adad6ca24 100644
--- a/crates/core/src/logstore/config.rs
+++ b/crates/core/src/logstore/config.rs
@@ -16,7 +16,7 @@ use super::storage::runtime::RuntimeConfig;
 use super::storage::LimitConfig;
 use crate::{DeltaResult, DeltaTableError};
 
-pub(super) trait TryUpdateKey {
+pub trait TryUpdateKey: Default {
     /// Update an internal field in the configuration.
     ///
     /// ## Returns
@@ -25,20 +25,17 @@ pub(super) trait TryUpdateKey {
     /// - `Err(_)` if the update failed. Failed updates may include finding a known key,
     ///   but failing to parse the value into the expected type.
     fn try_update_key(&mut self, key: &str, value: &str) -> DeltaResult<Option<()>>;
-}
 
-impl<K, V> FromIterator<(K, V)> for RuntimeConfig
-where
-    K: AsRef<str> + Into<String>,
-    V: AsRef<str> + Into<String>,
-{
-    fn from_iter<I: IntoIterator<Item = (K, V)>>(iter: I) -> Self {
-        ParseResult::from_iter(iter).config
-    }
+    /// Load configuration values from environment variables
+    ///
+    /// For Option<T> fields, this will only set values that are None
+    /// For non-optional fields, environment variables will update the
+    /// value if the current value corresponds to the default value.
+    fn load_from_environment(&mut self) -> DeltaResult<()>;
 }
 
 /// Generic container for parsing configuration
-pub(super) struct ParseResult<T> {
+pub struct ParseResult<T> {
     /// Parsed configuration
     pub config: T,
     /// Unrecognized key value pairs.
@@ -63,7 +60,7 @@ impl<T> ParseResult<T> {
 
 impl<T, K, V> FromIterator<(K, V)> for ParseResult<T>
 where
-    T: TryUpdateKey + Default,
+    T: TryUpdateKey,
     K: AsRef<str> + Into<String>,
     V: AsRef<str> + Into<String>,
 {
@@ -247,32 +244,57 @@ where
     I: IntoIterator<Item = (K, V)>,
     K: AsRef<str> + Into<String>,
     V: AsRef<str> + Into<String>,
-    T: TryUpdateKey + Default,
+    T: TryUpdateKey,
 {
     let result = ParseResult::from_iter(options);
     result.raise_errors()?;
     Ok((result.config, result.unparsed))
 }
 
-pub(super) fn parse_usize(value: &str) -> DeltaResult<usize> {
+pub fn parse_usize(value: &str) -> DeltaResult<usize> {
     value
         .parse::<usize>()
         .map_err(|_| DeltaTableError::Generic(format!("failed to parse \"{value}\" as usize")))
 }
 
-pub(super) fn parse_f64(value: &str) -> DeltaResult<f64> {
+pub fn parse_f64(value: &str) -> DeltaResult<f64> {
     value
         .parse::<f64>()
         .map_err(|_| DeltaTableError::Generic(format!("failed to parse \"{value}\" as f64")))
 }
 
-pub(super) fn parse_duration(value: &str) -> DeltaResult<std::time::Duration> {
+pub fn parse_duration(value: &str) -> DeltaResult<std::time::Duration> {
     humantime::parse_duration(value)
         .map_err(|_| DeltaTableError::Generic(format!("failed to parse \"{value}\" as Duration")))
 }
 
-pub(super) fn parse_bool(value: &str) -> DeltaResult<bool> {
-    Ok(super::storage::utils::str_is_truthy(value))
+pub fn parse_bool(value: &str) -> DeltaResult<bool> {
+    Ok(str_is_truthy(value))
+}
+
+pub fn parse_string(value: &str) -> DeltaResult<String> {
+    Ok(value.to_string())
+}
+
+/// Return true for all the stringly values typically associated with true
+///
+/// aka YAML booleans
+///
+/// ```rust
+/// # use deltalake_core::logstore::config::*;
+/// for value in ["1", "true", "on", "YES", "Y"] {
+///     assert!(str_is_truthy(value));
+/// }
+/// for value in ["0", "FALSE", "off", "NO", "n", "bork"] {
+///     assert!(!str_is_truthy(value));
+/// }
+/// ```
+pub fn str_is_truthy(val: &str) -> bool {
+    val.eq_ignore_ascii_case("1")
+        | val.eq_ignore_ascii_case("true")
+        | val.eq_ignore_ascii_case("on")
+        | val.eq_ignore_ascii_case("yes")
+        | val.eq_ignore_ascii_case("y")
 }
 
 #[cfg(all(test, feature = "cloud"))]
diff --git a/crates/core/src/logstore/mod.rs b/crates/core/src/logstore/mod.rs
index ecdb93fb7..21d84ff0d 100644
--- a/crates/core/src/logstore/mod.rs
+++ b/crates/core/src/logstore/mod.rs
@@ -78,7 +78,7 @@ pub use self::factories::{
     logstore_factories, object_store_factories, store_for, LogStoreFactory,
     LogStoreFactoryRegistry, ObjectStoreFactory, ObjectStoreFactoryRegistry,
 };
-pub use self::storage::utils::{commit_uri_from_version, str_is_truthy};
+pub use self::storage::utils::commit_uri_from_version;
 pub use self::storage::{
     DefaultObjectStoreRegistry, DeltaIOStorageBackend, IORuntime, ObjectStoreRef,
     ObjectStoreRegistry, ObjectStoreRetryExt,
@@ -86,7 +86,7 @@ pub use self::storage::{
 /// Convenience re-export of the object store crate
 pub use ::object_store;
 
-mod config;
+pub mod config;
 pub(crate) mod default_logstore;
 pub(crate) mod factories;
 pub(crate) mod storage;
diff --git a/crates/core/src/logstore/storage/mod.rs b/crates/core/src/logstore/storage/mod.rs
index cc61f9d03..f3db2dabf 100644
--- a/crates/core/src/logstore/storage/mod.rs
+++ b/crates/core/src/logstore/storage/mod.rs
@@ -6,8 +6,8 @@ use object_store::path::Path;
 use object_store::{DynObjectStore, ObjectStore};
 use url::Url;
 
-use super::config;
 use crate::{DeltaResult, DeltaTableError};
+use deltalake_derive::DeltaConfig;
 
 pub use retry_ext::ObjectStoreRetryExt;
 pub use runtime::{DeltaIOStorageBackend, IORuntime};
@@ -98,22 +98,88 @@ impl ObjectStoreRegistry for DefaultObjectStoreRegistry {
     }
 }
 
-#[derive(Debug, Clone, Default)]
+#[derive(Debug, Clone, Default, DeltaConfig)]
 pub struct LimitConfig {
+    #[delta(alias = "concurrency_limit", env = "OBJECT_STORE_CONCURRENCY_LIMIT")]
     pub max_concurrency: Option<usize>,
 }
 
-impl config::TryUpdateKey for LimitConfig {
-    fn try_update_key(&mut self, key: &str, v: &str) -> DeltaResult<Option<()>> {
-        match key {
-            // The number of concurrent connections the underlying object store can create
-            // Reference [LimitStore](https://docs.rs/object_store/latest/object_store/limit/struct.LimitStore.html)
-            // for more information
-            "OBJECT_STORE_CONCURRENCY_LIMIT" | "concurrency_limit" => {
-                self.max_concurrency = Some(config::parse_usize(v)?);
+#[cfg(test)]
+#[allow(drop_bounds, unused)]
+mod tests {
+    use std::collections::HashMap;
+    use std::env;
+
+    use rstest::*;
+
+    use super::*;
+    use crate::logstore::config::TryUpdateKey;
+
+    #[fixture]
+    pub fn with_env(#[default(vec![])] vars: Vec<(&str, &str)>) -> impl Drop {
+        // Store the original values before modifying
+        let original_values: HashMap<String, Option<String>> = vars
+            .iter()
+            .map(|(key, _)| (key.to_string(), std::env::var(key).ok()))
+            .collect();
+
+        // Set all the new environment variables
+        for (key, value) in vars {
+            std::env::set_var(key, value);
+        }
+
+        // Create a cleanup struct that will restore original values when dropped
+        struct EnvCleanup(HashMap<String, Option<String>>);
+
+        impl Drop for EnvCleanup {
+            fn drop(&mut self) {
+                for (key, maybe_value) in self.0.iter() {
+                    match maybe_value {
+                        Some(value) => env::set_var(key, value),
+                        None => env::remove_var(key),
+                    }
+                }
             }
-            _ => return Ok(None),
         }
-        Ok(Some(()))
+
+        EnvCleanup(original_values)
+    }
+
+    #[rstest]
+    fn test_api_with_env(
+        #[with(vec![("API_KEY", "test_key"), ("API_URL", "http://test.example.com")])]
+        with_env: impl Drop,
+    ) {
+        // Test code using these environment variables
+        assert_eq!(env::var("API_KEY").unwrap(), "test_key");
+        assert_eq!(env::var("API_URL").unwrap(), "http://test.example.com");
+
+        drop(with_env);
+
+        assert!(env::var("API_KEY").is_err());
+        assert!(env::var("API_URL").is_err());
+    }
+
+    #[test]
+    fn test_limit_config() {
+        let mut config = LimitConfig::default();
+        assert!(config.max_concurrency.is_none());
+
+        config.try_update_key("concurrency_limit", "10").unwrap();
+        assert_eq!(config.max_concurrency, Some(10));
+
+        config.try_update_key("max_concurrency", "20").unwrap();
+        assert_eq!(config.max_concurrency, Some(20));
+    }
+
+    #[rstest]
+    fn test_limit_config_env(
+        #[with(vec![("OBJECT_STORE_CONCURRENCY_LIMIT", "100")])] with_env: impl Drop,
+    ) {
+        let mut config = LimitConfig::default();
+        assert!(config.max_concurrency.is_none());
+
+        config.load_from_environment().unwrap();
+        assert_eq!(config.max_concurrency, Some(100));
     }
 }
diff --git a/crates/core/src/logstore/storage/retry_ext.rs b/crates/core/src/logstore/storage/retry_ext.rs
index e60033cdb..702861546 100644
--- a/crates/core/src/logstore/storage/retry_ext.rs
+++ b/crates/core/src/logstore/storage/retry_ext.rs
@@ -96,4 +96,8 @@ impl config::TryUpdateKey for object_store::RetryConfig {
         }
         Ok(Some(()))
     }
+
+    fn load_from_environment(&mut self) -> crate::DeltaResult<()> {
+        Ok(())
+    }
 }
diff --git a/crates/core/src/logstore/storage/runtime.rs b/crates/core/src/logstore/storage/runtime.rs
index b843fd499..a69e0d45f 100644
--- a/crates/core/src/logstore/storage/runtime.rs
+++ b/crates/core/src/logstore/storage/runtime.rs
@@ -2,6 +2,7 @@ use std::ops::Range;
 use std::sync::OnceLock;
 
 use bytes::Bytes;
+use deltalake_derive::DeltaConfig;
 use futures::future::BoxFuture;
 use futures::stream::BoxStream;
 use futures::FutureExt;
@@ -15,9 +16,6 @@ use object_store::{MultipartUpload, PutMultipartOpts};
 use serde::{Deserialize, Serialize};
 use tokio::runtime::{Builder as RuntimeBuilder, Handle, Runtime};
 
-use crate::logstore::config;
-use crate::DeltaResult;
-
 /// Creates static IO Runtime with optional configuration
 fn io_rt(config: Option<&RuntimeConfig>) -> &Runtime {
     static IO_RT: OnceLock<Runtime> = OnceLock::new();
@@ -66,7 +64,7 @@ fn io_rt(config: Option<&RuntimeConfig>) -> &Runtime {
 }
 
 /// Configuration for Tokio runtime
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+#[derive(Debug, Clone, Serialize, Deserialize, Default, DeltaConfig)]
 pub struct RuntimeConfig {
     /// Whether to use a multi-threaded runtime
     pub(crate) multi_threaded: Option<bool>,
@@ -80,20 +78,6 @@ pub struct RuntimeConfig {
     pub(crate) enable_time: Option<bool>,
 }
 
-impl config::TryUpdateKey for RuntimeConfig {
-    fn try_update_key(&mut self, key: &str, v: &str) -> DeltaResult<Option<()>> {
-        match key {
-            "multi_threaded" => self.multi_threaded = Some(config::parse_bool(v)?),
-            "worker_threads" => self.worker_threads = Some(config::parse_usize(v)?),
-            "thread_name" => self.thread_name = Some(v.into()),
-            "enable_io" => self.enable_io = Some(config::parse_bool(v)?),
-            "enable_time" => self.enable_time = Some(config::parse_bool(v)?),
-            _ => return Ok(None),
-        }
-        Ok(Some(()))
-    }
-}
-
 impl RuntimeConfig {
     pub fn decorate<T: ObjectStore + Clone>(
         &self,
diff --git a/crates/core/src/logstore/storage/utils.rs b/crates/core/src/logstore/storage/utils.rs
index 1487c7dac..ab4052c01 100644
--- a/crates/core/src/logstore/storage/utils.rs
+++ b/crates/core/src/logstore/storage/utils.rs
@@ -7,27 +7,6 @@ use object_store::ObjectMeta;
 use crate::errors::{DeltaResult, DeltaTableError};
 use crate::kernel::Add;
 
-/// Return true for all the stringly values typically associated with true
-///
-/// aka YAML booleans
-///
-/// ```rust
-/// # use deltalake_core::logstore::*;
-/// for value in ["1", "true", "on", "YES", "Y"] {
-///     assert!(str_is_truthy(value));
-/// }
-/// for value in ["0", "FALSE", "off", "NO", "n", "bork"] {
-///     assert!(!str_is_truthy(value));
-/// }
-/// ```
-pub fn str_is_truthy(val: &str) -> bool {
-    val.eq_ignore_ascii_case("1")
-        | val.eq_ignore_ascii_case("true")
-        | val.eq_ignore_ascii_case("on")
-        | val.eq_ignore_ascii_case("yes")
-        | val.eq_ignore_ascii_case("y")
-}
-
 /// Return the uri of commit version.
 ///
 /// ```rust
diff --git a/crates/core/src/table/builder.rs b/crates/core/src/table/builder.rs
index cd7025b71..4e7c96478 100644
--- a/crates/core/src/table/builder.rs
+++ b/crates/core/src/table/builder.rs
@@ -5,6 +5,7 @@ use std::path::PathBuf;
 use std::sync::Arc;
 
 use chrono::{DateTime, FixedOffset, Utc};
+use deltalake_derive::DeltaConfig;
 use object_store::DynObjectStore;
 use serde::{Deserialize, Serialize};
 use tracing::debug;
@@ -27,7 +28,7 @@ pub enum DeltaVersion {
 }
 
 /// Configuration options for delta table
-#[derive(Debug, Serialize, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone, DeltaConfig)]
 #[serde(rename_all = "camelCase")]
 pub struct DeltaTableConfig {
     /// Indicates whether DeltaTable should track files.
@@ -51,6 +52,7 @@ pub struct DeltaTableConfig {
     pub log_batch_size: usize,
 
     #[serde(skip_serializing, skip_deserializing)]
+    #[delta(skip)]
     /// When a runtime handler is provided, all IO tasks are spawn in that handle
     pub io_runtime: Option<IORuntime>,
 }
diff --git a/crates/derive/Cargo.toml b/crates/derive/Cargo.toml
new file mode 100644
index 000000000..4d1c35f32
--- /dev/null
+++ b/crates/derive/Cargo.toml
@@ -0,0 +1,23 @@
+[package]
+name = "deltalake-derive"
+version = "0.25.0"
+description = "Dervice macoros for use in delta ecosystem crates"
+authors.workspace = true
+rust-version.workspace = true
+keywords.workspace = true
+readme.workspace = true
+edition.workspace = true
+homepage.workspace = true
+license.workspace = true
+documentation.workspace = true
+repository.workspace = true
+
+[lib]
+proc-macro = true
+
+[dependencies]
+proc-macro2 = "1"
+syn = { version = "2.0", features = ["extra-traits"] }
+quote = "1.0"
+convert_case = "0.8.0"
+itertools = "0.14"
diff --git a/crates/derive/src/lib.rs b/crates/derive/src/lib.rs
new file mode 100644
index 000000000..3c5b38af5
--- /dev/null
+++ b/crates/derive/src/lib.rs
@@ -0,0 +1,356 @@
+use convert_case::{Case, Casing};
+use itertools::Itertools;
+use proc_macro::TokenStream;
+use proc_macro2::Span;
+use quote::quote;
+use syn::parse::Parser;
+use syn::{
+    parse_macro_input, punctuated::Punctuated, Data, DeriveInput, Field, Fields, Lit, Meta,
+    MetaNameValue, Token, Type,
+};
+use syn::{Attribute, Error, Expr, Ident, Result};
+
+/// Derive macro for implementing the TryUpdateKey trait
+///
+/// This macro automatically implements TryUpdateKey for a struct,
+/// mapping field names to configuration keys and using appropriate parsers
+/// based on the field type.
+///
+/// Additional key aliases can be specified with the `#[delta(alias = "alias.name")]` attribute.
+/// Multiple aliases can be added by `#[delta(alias = "foo", alias = "bar")]`.
+///
+/// Reading configuration can be achieved by assigning environmene keys to a field
+/// `#[delta(env = "MY_ENV_KEY")]`.
+#[proc_macro_derive(DeltaConfig, attributes(delta))]
+pub fn derive_delta_config(input: TokenStream) -> TokenStream {
+    // Parse the input tokens into a syntax tree
+    let input = parse_macro_input!(input as DeriveInput);
+
+    // Get the name of the struct
+    let name = &input.ident;
+
+    // Extract the fields from the struct
+    let fields = match &input.data {
+        Data::Struct(data) => match &data.fields {
+            Fields::Named(fields) => &fields.named,
+            _ => panic!("TryUpdateKey can only be derived for structs with named fields"),
+        },
+        _ => panic!("TryUpdateKey can only be derived for structs"),
+    }
+    .into_iter()
+    .collect::<Vec<_>>();
+
+    // Generate the implementation for TryUpdateKey trait
+    let try_update_key = match generate_try_update_key(name, &fields) {
+        Ok(try_update_key) => try_update_key,
+        Err(err) => return syn::Error::into_compile_error(err).into(),
+    };
+
+    // generate an enum with all configuration keys
+    let config_keys = match generate_config_keys(name, &fields) {
+        Ok(config_keys) => config_keys,
+        Err(err) => return syn::Error::into_compile_error(err).into(),
+    };
+
+    // generate a FromIterator implementation
+    let from_iter = generate_from_iterator(name);
+
+    let expanded = quote! {
+        #try_update_key
+
+        #config_keys
+
+        #from_iter
+    };
+
+    TokenStream::from(expanded)
+}
+
+fn generate_config_keys(name: &Ident, fields: &[&Field]) -> Result<proc_macro2::TokenStream> {
+    let enum_name = Ident::new(&format!("{}Key", name), Span::call_site());
+    let variants: Vec<_> = fields
+        .iter()
+        .map(|field| {
+            let field_name = &field
+                .ident
+                .as_ref()
+                .ok_or_else(|| syn::Error::new_spanned(field, "expected name"))?
+                .to_string();
+            let pascal_case = Ident::new(&field_name.to_case(Case::Pascal), Span::call_site());
+            let attributes = extract_field_attributes(&field.attrs)?;
+
+            // Generate doc attribute if documentation exists
+            let doc_attr = if let Some(doc_string) = attributes.docs {
+                // Create a doc attribute for the enum variant
+                quote! { #[doc = #doc_string] }
+            } else {
+                // No documentation
+                quote! {}
+            };
+
+            // Return the variant with its documentation
+            Ok(quote! {
+                #doc_attr
+                #pascal_case
+            })
+        })
+        .collect::<Result<_>>()?;
+    Ok(quote! {
+        #[automatically_derived]
+        pub enum #enum_name {
+            #(#variants),*
+        }
+    })
+}
+
+fn generate_from_iterator(name: &Ident) -> proc_macro2::TokenStream {
+    quote! {
+        #[automatically_derived]
+        impl<K, V> FromIterator<(K, V)> for #name
+        where
+            K: AsRef<str> + Into<String>,
+            V: AsRef<str> + Into<String>,
+        {
+            fn from_iter<I: IntoIterator<Item = (K, V)>>(iter: I) -> Self {
+                crate::logstore::config::ParseResult::from_iter(iter).config
+            }
+        }
+    }
+}
+
+fn generate_try_update_key(name: &Ident, fields: &[&Field]) -> Result<proc_macro2::TokenStream> {
+    let match_arms: Vec<_> = fields
+        .iter()
+        .filter_map(|field| {
+            let field_name = &field.ident.as_ref().unwrap();
+            let field_name_str = field_name.to_string();
+
+            // Extract aliases from attributes
+            let attributes = match extract_field_attributes(&field.attrs) {
+                Ok(attributes) => attributes,
+                Err(e) => return Some(Err(e)),
+            };
+            if attributes.skip {
+                return None;
+            }
+
+            // Determine parser based on field type
+            let (parser, is_option) = match determine_parser(&field.ty) {
+                Ok((parser, is_option)) => (parser, is_option),
+                Err(e) => return Some(Err(e)),
+            };
+
+            // Build the match conditions: field name and all aliases
+            let mut match_conditions = vec![quote! { #field_name_str }];
+            for alias in attributes.aliases {
+                match_conditions.push(quote! { #alias });
+            }
+
+            let match_arm = if is_option {
+                quote! {
+                    #(#match_conditions)|* => self.#field_name = Some(#parser(v)?),
+                }
+            } else {
+                quote! {
+                    #(#match_conditions)|* => self.#field_name = #parser(v)?,
+                }
+            };
+
+            Some(Ok(match_arm))
+        })
+        .try_collect()?;
+
+    let env_setters = generate_load_from_env(fields)?;
+
+    Ok(quote! {
+        #[automatically_derived]
+        impl crate::logstore::config::TryUpdateKey for #name {
+            fn try_update_key(&mut self, key: &str, v: &str) -> crate::DeltaResult<Option<()>> {
+                match key {
+                    #(#match_arms)*
+                    _ => return Ok(None),
+                }
+                Ok(Some(()))
+            }
+
+            fn load_from_environment(&mut self) -> crate::DeltaResult<()> {
+                let default_values = Self::default();
+                #(#env_setters)*
+                Ok(())
+            }
+        }
+    })
+}
+
+fn generate_load_from_env(fields: &[&Field]) -> Result<Vec<proc_macro2::TokenStream>> {
+    fields.iter().filter_map(|field| {
+        let field_name = &field.ident.as_ref().unwrap();
+        let attributes = match extract_field_attributes(&field.attrs) {
+            Ok(attributes) => attributes,
+            Err(e) => return Some(Err(e)),
+        };
+        if attributes.skip || attributes.env_variable_names.is_empty() {
+            return None;
+        }
+
+        let (parser, is_option) = match determine_parser(&field.ty) {
+            Ok((parser, is_option)) => (parser, is_option),
+            Err(e) => return Some(Err(e))
+        };
+
+        let env_checks = attributes.env_variable_names.iter().map(|env_var| {
+            if is_option {
+                // For Option types, only set if None
+                quote! {
+                    if self.#field_name.is_none() {
+                        if let Ok(val) = std::env::var(#env_var) {
+                            match #parser(&val) {
+                                Ok(parsed) => self.#field_name = Some(parsed),
+                                Err(e) => ::tracing::warn!("Failed to parse environment variable {}: {}", #env_var, e),
+                            }
+                        }
+                    }
+                }
+            } else {
+                // For non-Option types, we override the default value
+                // but ignore it if the current value is not the default.
+                quote! {
+                    if self.#field_name == default_values.#field_name {
+                        if let Ok(val) = std::env::var(#env_var) {
+                            match #parser(&val) {
+                                Ok(parsed) => self.#field_name = parsed,
+                                Err(e) => ::tracing::warn!("Failed to parse environment variable {}: {}", #env_var, e),
+                            }
+                        }
+                    }
+                }
+            }
+        });
+
+        Some(Ok(quote! {
+            #(#env_checks)*
+        }))
+    }).try_collect()
+}
+
+// Helper function to determine the appropriate parser based on field type
+fn determine_parser(ty: &Type) -> Result<(proc_macro2::TokenStream, bool)> {
+    match ty {
+        Type::Path(type_path) => {
+            let type_str = quote! { #type_path }.to_string();
+            let is_option = type_str.starts_with("Option");
+
+            let caller = if type_str.contains("usize") {
+                quote! { crate::logstore::config::parse_usize }
+            } else if type_str.contains("f64") || type_str.contains("f32") {
+                quote! { crate::logstore::config::parse_f64 }
+            } else if type_str.contains("Duration") {
+                quote! { crate::logstore::config::parse_duration }
+            } else if type_str.contains("bool") {
+                quote! { crate::logstore::config::parse_bool }
+            } else if type_str.contains("String") {
+                quote! { crate::logstore::config::parse_string }
+            } else {
+                return Err(Error::new_spanned(ty,
+                    format!("Unsupported field type: {type_str}. Consider implementing a custom parser.")
+                ));
+            };
+
+            Ok((caller, is_option))
+        }
+        _ => panic!("Unsupported field type for TryUpdateKey"),
+    }
+}
+
+struct FieldAttributes {
+    aliases: Vec<String>,
+    env_variable_names: Vec<String>,
+    docs: Option<String>,
+    skip: bool,
+}
+
+/// Extract aliases from field attributes
+///
+/// Parse the annotations from individual fields into a convenient structure.
+/// The field is annotated via `#[delta(...)]`. The following attributes are supported:
+/// - `alias = "alias_name"`: Specifies an alias for the field.
+/// - `env = "env_name"`: Specifies an environment variable name for the field.
+/// - `skip`: Specifies whether the field should be skipped during parsing.
+fn extract_field_attributes(attrs: &[Attribute]) -> Result<FieldAttributes> {
+    let mut aliases = Vec::new();
+    let mut environments = Vec::new();
+    let mut docs = None;
+    let mut doc_strings = Vec::new();
+    let mut skip = false;
+
+    for attr in attrs {
+        if attr.path().is_ident("doc") {
+            // Handle doc comments
+            let meta = attr.meta.require_name_value()?;
+            if let Expr::Lit(expr_lit) = &meta.value {
+                if let Lit::Str(lit_str) = &expr_lit.lit {
+                    // Collect all doc strings - they might span multiple lines
+                    doc_strings.push(lit_str.value().trim().to_string());
+                }
+            }
+        }
+        if attr.path().is_ident("delta") {
+            match &attr.meta {
+                Meta::List(list) => {
+                    let parser = Punctuated::<Meta, Token![,]>::parse_terminated;
+                    let parsed = parser.parse(list.tokens.clone().into())?;
+
+                    for val in parsed {
+                        match val {
+                            Meta::NameValue(MetaNameValue { path, value, .. }) => {
+                                if path.is_ident("alias") {
+                                    if let Expr::Lit(lit_expr) = &value {
+                                        if let Lit::Str(lit_str) = &lit_expr.lit {
+                                            aliases.push(lit_str.value());
+                                        }
+                                    }
+                                }
+                                if path.is_ident("environment") || path.is_ident("env") {
+                                    if let Expr::Lit(lit_expr) = &value {
+                                        if let Lit::Str(lit_str) = &lit_expr.lit {
+                                            environments.push(lit_str.value());
+                                        }
+                                    }
+                                }
+                            }
+                            Meta::Path(path) => {
+                                if path.is_ident("skip") {
+                                    skip = true;
+                                }
+                            }
+                            _ => {
+                                return Err(Error::new_spanned(
+                                    &attr.meta,
+                                    "only NameValue and Path parameters are supported",
+                                ));
+                            }
+                        }
+                    }
+                }
+                _ => {
+                    return Err(Error::new_spanned(
+                        &attr.meta,
+                        "expected a list-style attribute",
+                    ));
+                }
+            }
+        }
+    }
+
+    // Combine all doc strings into a single documentation string
+    if !doc_strings.is_empty() {
+        docs = Some(doc_strings.join("\n"));
+    }
+
+    Ok(FieldAttributes {
+        aliases,
+        env_variable_names: environments,
+        docs,
+        skip,
+    })
+}
diff --git a/crates/mount/src/lib.rs b/crates/mount/src/lib.rs
index d7057c7e2..a5ac35157 100644
--- a/crates/mount/src/lib.rs
+++ b/crates/mount/src/lib.rs
@@ -3,10 +3,8 @@ use std::str::FromStr;
 use std::sync::Arc;
 
 use deltalake_core::logstore::{
-    default_logstore, logstore_factories, LogStore, LogStoreFactory, StorageConfig,
-};
-use deltalake_core::logstore::{
-    object_store_factories, str_is_truthy, ObjectStoreFactory, ObjectStoreRef,
+    config::str_is_truthy, default_logstore, logstore_factories, object_store_factories, LogStore,
+    LogStoreFactory, ObjectStoreFactory, ObjectStoreRef, StorageConfig,
 };
 use deltalake_core::{DeltaResult, DeltaTableError, Path};
 use object_store::local::LocalFileSystem;


Reviews:
user: login: roeap
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-16T09:33:17Z
user: login: ion-elgreco
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-16T13:43:05Z
user: login: ion-elgreco
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-16T13:47:28Z
user: login: roeap
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-16T13:50:52Z
user: login: roeap
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-16T13:51:54Z
user: login: ion-elgreco
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-16T13:53:44Z
user: login: ion-elgreco
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-16T20:53:13Z

Review comments:
diff_hunk: @@ -98,22 +98,88 @@ impl ObjectStoreRegistry for DefaultObjectStoreRegistry {
     }
 }
 
-#[derive(Debug, Clone, Default)]
+#[derive(Debug, Clone, Default, DeltaConfig)]
 pub struct LimitConfig {
+    #[delta(alias = "concurrency_limit", env = "OBJECT_STORE_CONCURRENCY_LIMIT")]
     pub max_concurrency: Option<usize>,
 }
 
-impl config::TryUpdateKey for LimitConfig {
-    fn try_update_key(&mut self, key: &str, v: &str) -> DeltaResult<Option<()>> {
-        match key {
-            // The number of concurrent connections the underlying object store can create
-            // Reference [LimitStore](https://docs.rs/object_store/latest/object_store/limit/struct.LimitStore.html)
-            // for more information
-            "OBJECT_STORE_CONCURRENCY_LIMIT" | "concurrency_limit" => {
-                self.max_concurrency = Some(config::parse_usize(v)?);
+#[cfg(test)]
+#[allow(drop_bounds, unused)]
+mod tests {
+    use std::collections::HashMap;
+    use std::env;
+
+    use rstest::*;
path: crates/core/src/logstore/storage/mod.rs
user: login: roeap
body: Seems like its been too long that I looked into test frameworks for rust. Learned about `rstest` with this PR and I think using fixtures and other mechanics it offers, we can greatly reduce some of our boilerplate.

The kernel PR also adds dat tests for rust, which I want to carve out and use as an opportunity to maybe refactor some of our test scaffolding.
created_at: 2025-04-16T09:33:17Z
author_association: COLLABORATOR
diff_hunk: @@ -98,22 +98,88 @@ impl ObjectStoreRegistry for DefaultObjectStoreRegistry {
     }
 }
 
-#[derive(Debug, Clone, Default)]
+#[derive(Debug, Clone, Default, DeltaConfig)]
 pub struct LimitConfig {
+    #[delta(alias = "concurrency_limit", env = "OBJECT_STORE_CONCURRENCY_LIMIT")]
     pub max_concurrency: Option<usize>,
 }
 
-impl config::TryUpdateKey for LimitConfig {
-    fn try_update_key(&mut self, key: &str, v: &str) -> DeltaResult<Option<()>> {
-        match key {
-            // The number of concurrent connections the underlying object store can create
-            // Reference [LimitStore](https://docs.rs/object_store/latest/object_store/limit/struct.LimitStore.html)
-            // for more information
-            "OBJECT_STORE_CONCURRENCY_LIMIT" | "concurrency_limit" => {
-                self.max_concurrency = Some(config::parse_usize(v)?);
+#[cfg(test)]
+#[allow(drop_bounds, unused)]
+mod tests {
+    use std::collections::HashMap;
+    use std::env;
+
+    use rstest::*;
+
+    use super::*;
+    use crate::logstore::config::TryUpdateKey;
+
+    #[fixture]
+    pub fn with_env(#[default(vec![])] vars: Vec<(&str, &str)>) -> impl Drop {
path: crates/core/src/logstore/storage/mod.rs
user: login: ion-elgreco
body: Oh this is cool!
created_at: 2025-04-16T13:43:04Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,356 @@
+use convert_case::{Case, Casing};
+use itertools::Itertools;
+use proc_macro::TokenStream;
+use proc_macro2::Span;
+use quote::quote;
+use syn::parse::Parser;
+use syn::{
+    parse_macro_input, punctuated::Punctuated, Data, DeriveInput, Field, Fields, Lit, Meta,
+    MetaNameValue, Token, Type,
+};
+use syn::{Attribute, Error, Expr, Ident, Result};
+
+/// Derive macro for implementing the TryUpdateKey trait
+///
+/// This macro automatically implements TryUpdateKey for a struct,
+/// mapping field names to configuration keys and using appropriate parsers
+/// based on the field type.
+///
+/// Additional key aliases can be specified with the `#[delta(alias = "alias.name")]` attribute.
+/// Multiple aliases can be added by `#[delta(alias = "foo", alias = "bar")]`.
+///
+/// Reading configuration can be achieved by assigning environmene keys to a field
+/// `#[delta(env = "MY_ENV_KEY")]`.
+#[proc_macro_derive(DeltaConfig, attributes(delta))]
path: crates/derive/src/lib.rs
user: login: ion-elgreco
body: I fully trust your dark magic here, but maybe it makes sense to move those derive tests to this crate
created_at: 2025-04-16T13:47:28Z
author_association: COLLABORATOR
diff_hunk: @@ -98,22 +98,88 @@ impl ObjectStoreRegistry for DefaultObjectStoreRegistry {
     }
 }
 
-#[derive(Debug, Clone, Default)]
+#[derive(Debug, Clone, Default, DeltaConfig)]
 pub struct LimitConfig {
+    #[delta(alias = "concurrency_limit", env = "OBJECT_STORE_CONCURRENCY_LIMIT")]
     pub max_concurrency: Option<usize>,
 }
 
-impl config::TryUpdateKey for LimitConfig {
-    fn try_update_key(&mut self, key: &str, v: &str) -> DeltaResult<Option<()>> {
-        match key {
-            // The number of concurrent connections the underlying object store can create
-            // Reference [LimitStore](https://docs.rs/object_store/latest/object_store/limit/struct.LimitStore.html)
-            // for more information
-            "OBJECT_STORE_CONCURRENCY_LIMIT" | "concurrency_limit" => {
-                self.max_concurrency = Some(config::parse_usize(v)?);
+#[cfg(test)]
+#[allow(drop_bounds, unused)]
+mod tests {
+    use std::collections::HashMap;
+    use std::env;
+
+    use rstest::*;
+
+    use super::*;
+    use crate::logstore::config::TryUpdateKey;
+
+    #[fixture]
+    pub fn with_env(#[default(vec![])] vars: Vec<(&str, &str)>) -> impl Drop {
path: crates/core/src/logstore/storage/mod.rs
user: login: roeap
body: I thought so as well :) - there is tons more stuff this library can do for us!
created_at: 2025-04-16T13:50:52Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,356 @@
+use convert_case::{Case, Casing};
+use itertools::Itertools;
+use proc_macro::TokenStream;
+use proc_macro2::Span;
+use quote::quote;
+use syn::parse::Parser;
+use syn::{
+    parse_macro_input, punctuated::Punctuated, Data, DeriveInput, Field, Fields, Lit, Meta,
+    MetaNameValue, Token, Type,
+};
+use syn::{Attribute, Error, Expr, Ident, Result};
+
+/// Derive macro for implementing the TryUpdateKey trait
+///
+/// This macro automatically implements TryUpdateKey for a struct,
+/// mapping field names to configuration keys and using appropriate parsers
+/// based on the field type.
+///
+/// Additional key aliases can be specified with the `#[delta(alias = "alias.name")]` attribute.
+/// Multiple aliases can be added by `#[delta(alias = "foo", alias = "bar")]`.
+///
+/// Reading configuration can be achieved by assigning environmene keys to a field
+/// `#[delta(env = "MY_ENV_KEY")]`.
+#[proc_macro_derive(DeltaConfig, attributes(delta))]
path: crates/derive/src/lib.rs
user: login: roeap
body: 😆 - unfortunately i cannot ... you can't use the derive macros within the crate they are defined in. There are some hacks using doc-tests, but these are hacky ...
created_at: 2025-04-16T13:51:54Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,356 @@
+use convert_case::{Case, Casing};
+use itertools::Itertools;
+use proc_macro::TokenStream;
+use proc_macro2::Span;
+use quote::quote;
+use syn::parse::Parser;
+use syn::{
+    parse_macro_input, punctuated::Punctuated, Data, DeriveInput, Field, Fields, Lit, Meta,
+    MetaNameValue, Token, Type,
+};
+use syn::{Attribute, Error, Expr, Ident, Result};
+
+/// Derive macro for implementing the TryUpdateKey trait
+///
+/// This macro automatically implements TryUpdateKey for a struct,
+/// mapping field names to configuration keys and using appropriate parsers
+/// based on the field type.
+///
+/// Additional key aliases can be specified with the `#[delta(alias = "alias.name")]` attribute.
+/// Multiple aliases can be added by `#[delta(alias = "foo", alias = "bar")]`.
+///
+/// Reading configuration can be achieved by assigning environmene keys to a field
+/// `#[delta(env = "MY_ENV_KEY")]`.
+#[proc_macro_derive(DeltaConfig, attributes(delta))]
path: crates/derive/src/lib.rs
user: login: ion-elgreco
body: Oh that's strange, I didn't know that xd
created_at: 2025-04-16T13:53:44Z
author_association: COLLABORATOR

PR comments:
user: login: github-actions[bot]
created_at: 2025-04-16T09:29:24Z
body: **ACTION NEEDED**
  
  delta-rs follows the [Conventional Commits  specification](https://www.conventionalcommits.org/en/v1.0.0/) for  release automation.

  The PR title and description are used as the merge commit message.  Please update your PR title and description to match the specification.
  
author_association: NONE
user: login: codecov[bot]
created_at: 2025-04-16T13:59:03Z
body: ## [Codecov](https://app.codecov.io/gh/delta-io/delta-rs/pull/3389?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=delta-io) Report
Attention: Patch coverage is `77.69784%` with `62 lines` in your changes missing coverage. Please review.
> Project coverage is 71.96%. Comparing base [(`41b4265`)](https://app.codecov.io/gh/delta-io/delta-rs/commit/41b42652601a190c52601f21d230b755470d468b?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=delta-io) to head [(`034552b`)](https://app.codecov.io/gh/delta-io/delta-rs/commit/034552baa58b7c08ea0c7c511a267fb4d3121c73?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=delta-io).
> Report is 4 commits behind head on main.

| [Files with missing lines](https://app.codecov.io/gh/delta-io/delta-rs/pull/3389?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=delta-io) | Patch % | Lines |
|---|---|---|
| [crates/derive/src/lib.rs](https://app.codecov.io/gh/delta-io/delta-rs/pull/3389?src=pr&el=tree&filepath=crates%2Fderive%2Fsrc%2Flib.rs&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=delta-io#diff-Y3JhdGVzL2Rlcml2ZS9zcmMvbGliLnJz) | 80.99% | [35 Missing and 11 partials :warning: ](https://app.codecov.io/gh/delta-io/delta-rs/pull/3389?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=delta-io) |
| [crates/core/src/logstore/config.rs](https://app.codecov.io/gh/delta-io/delta-rs/pull/3389?src=pr&el=tree&filepath=crates%2Fcore%2Fsrc%2Flogstore%2Fconfig.rs&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=delta-io#diff-Y3JhdGVzL2NvcmUvc3JjL2xvZ3N0b3JlL2NvbmZpZy5ycw==) | 25.00% | [12 Missing :warning: ](https://app.codecov.io/gh/delta-io/delta-rs/pull/3389?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=delta-io) |
| [crates/core/src/logstore/storage/retry\_ext.rs](https://app.codecov.io/gh/delta-io/delta-rs/pull/3389?src=pr&el=tree&filepath=crates%2Fcore%2Fsrc%2Flogstore%2Fstorage%2Fretry_ext.rs&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=delta-io#diff-Y3JhdGVzL2NvcmUvc3JjL2xvZ3N0b3JlL3N0b3JhZ2UvcmV0cnlfZXh0LnJz) | 0.00% | [3 Missing :warning: ](https://app.codecov.io/gh/delta-io/delta-rs/pull/3389?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=delta-io) |
| [crates/core/src/logstore/storage/mod.rs](https://app.codecov.io/gh/delta-io/delta-rs/pull/3389?src=pr&el=tree&filepath=crates%2Fcore%2Fsrc%2Flogstore%2Fstorage%2Fmod.rs&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=delta-io#diff-Y3JhdGVzL2NvcmUvc3JjL2xvZ3N0b3JlL3N0b3JhZ2UvbW9kLnJz) | 94.11% | [1 Missing :warning: ](https://app.codecov.io/gh/delta-io/delta-rs/pull/3389?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=delta-io) |

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main    #3389      +/-   ##
==========================================
+ Coverage   71.88%   71.96%   +0.08%     
==========================================
  Files         147      148       +1     
  Lines       45840    46083     +243     
  Branches    45840    46083     +243     
==========================================
+ Hits        32951    33163     +212     
- Misses      10781    10819      +38     
+ Partials     2108     2101       -7     
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/delta-io/delta-rs/pull/3389?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=delta-io).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=delta-io).

<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
- :package: [JS Bundle Analysis](https://docs.codecov.com/docs/javascript-bundle-analysis): Save yourself from yourself by tracking and limiting bundle sizes in JS merges.
</details>
author_association: NONE