Package: config_wrangler
Config Wranglerpydantic based configuration wrangler. Handles reading multiple ini or toml files with inheritance rules and variable expansions.InstallationInstall using your package manager of choice:poetry add config-wranglerpip install -U config-wranglerconda install config-wrangler -c conda-forge.A Simple Exampleconfig.ini[S3_Source]bucket_name=my.exmple-bucketkey_prefixes=processed/user_id=AK123456789ABC# Not a secure way to store the password, but OK for local prototype or examples.# See KEYRING or KEEPASS for better optionspassword_source=CONFIG_FILEraw_password=My secret password[target_database]dialect=sqlitedatabase_name=${test_section:my_environment:source_data_dir}/example_db[test_section]my_int=123my_float=123.45my_bool=Yesmy_str=ABC☕my_bytes=ABCⓁⓄⓋ☕my_list_auto_c=a,b,cmy_list_auto_nl=abcmy_list_auto_pipe=a|b|cmy_list_c=a,b,cmy_list_python=['x','y','z']my_list_json=["J","S","O","N"]my_list_nl=abcmy_list_int_c=1,2,3my_tuple_c=a,b,cmy_tuple_nl=abcmy_tuple_int_c=1,2,3my_dict={1: "One", 2: "Two"}my_dict_str_int={"one": 1, "two": 2}my_set={'A','B','C'}my_set_int=1,2,3my_frozenset=A,B,Cmy_date=2021-05-31my_time=11:55:23my_datetime=2021-05-31 11:23:53my_url=https://localhost:6553/[test_section.my_environment]name=dev# For example to run we'll make both paths relative to currenttemp_data_dir=.\temp_data\${test_section:my_environment:name}source_data_dir=.python codeimporttypingfromdatetimeimportdate,time,datetimefrompydanticimportBaseModel,DirectoryPath,Field,AnyHttpUrlfromconfig_wrangler.config_data_loaders.base_config_data_loaderimportBaseConfigDataLoaderfromconfig_wrangler.config_from_ini_envimportConfigFromIniEnvfromconfig_wrangler.config_from_loadersimportConfigFromLoadersfromconfig_wrangler.config_templates.config_hierarchyimportConfigHierarchyfromconfig_wrangler.config_templates.aws.s3_bucketimportS3_Bucketfromconfig_wrangler.config_templates.sqlalchemy_databaseimportSQLAlchemyDatabasefromconfig_wrangler.config_types.path_typesimportAutoCreateDirectoryPathfromconfig_wrangler.config_types.delimited_fieldimportDelimitedListFieldclassS3_Bucket_KeyPrefixes(S3_Bucket):key_prefixes:typing.List[str]classEnvironment(ConfigHierarchy):name:str=Field(...,env='env_name')temp_data_dir:AutoCreateDirectoryPathsource_data_dir:DirectoryPathclassTestSection(BaseModel):my_int:intmy_float:floatmy_bool:boolmy_str:strmy_bytes:bytesmy_list_auto_c:listmy_list_auto_nl:listmy_list_auto_pipe:listmy_list_python:listmy_list_json:listmy_list_c:list=DelimitedListField(delimiter=',')my_list_nl:list=DelimitedListField(delimiter='\n')my_list_int_c:typing.List[int]=DelimitedListField(delimiter=',')my_tuple_c:tuple=DelimitedListField(delimiter=',')my_tuple_nl:tuple=DelimitedListField(delimiter='\n')my_tuple_int_c:typing.Tuple[int,int,int]=DelimitedListField(delimiter=',')my_dict:dictmy_dict_str_int:typing.Dict[str,int]my_set:setmy_set_int:typing.Set[int]my_frozenset:frozensetmy_date:datemy_time:timemy_datetime:datetimemy_url:AnyHttpUrlmy_environment:EnvironmentclassETLConfig(ConfigFromIniEnv):classConfig:validate_default=Truevalidate_assignment=Truetarget_database:SQLAlchemyDatabases3_source:S3_Bucket_KeyPrefixestest_section:TestSectionclassETLConfigAnyLoaders(ETLConfig):def__init__(self,_config_data_loaders:typing.List[BaseConfigDataLoader],**kwargs:typing.Dict[str,typing.Any])->None:# Skip super and call the next higher classConfigFromLoaders.__init__(self,_config_data_loaders=_config_data_loaders,**kwargs)defmain():config=ETLConfig(file_name='simple_example.ini')print(f"Temp data dir ={config.test_section.my_environment.temp_data_dir}")# > Temp data dir = temp_data\devprint(f"Source data dir ={config.test_section.my_environment.source_data_dir}")# > Source data dir = .print(f"my_int ={config.test_section.my_int}")# > my_int = 123print(f"my_float ={config.test_section.my_float}")# > my_float = 123.45print(f"my_str ={config.test_section.my_str}")# > my_str = ABC☕print(f"my_list_auto_c ={config.test_section.my_list_auto_c}")# > my_list_auto_c = ['a', 'b', 'c']print(f"my_list_auto_nl ={config.test_section.my_list_auto_nl}")# > my_list_auto_c = ['a', 'b', 'c']print(f"my_dict ={config.test_section.my_dict}")# > my_dict = {1: 'One', 2: 'Two'}print(f"my_set ={config.test_section.my_set}")# > my_set = {'C', 'A', 'B'}print(f"my_time ={config.test_section.my_time}")# > my_time = 11:55:23print(f"my_datetime ={config.test_section.my_datetime}")# > my_datetime = 2021-05-31 11:23:53print(f"my_url ={config.test_section.my_url}")# > my_url = https://localhost:6553/# Getting DB engine (requires sqlalchemy optional installengine=config.target_database.get_engine()print(f"target_database.engine ={engine}")# > target_database.engine = Engine(sqlite:///.example_db)print("Getting S3 Data")bucket=config.s3_source.get_bucket()print(f"S3 bucket definition ={bucket}")forprefixinconfig.s3_source.key_prefixes:print(f"  bucket search prefix ={prefix}")# > Getting S3 Data# > credentials.py:56: UserWarning: Passwords stored directly in config or worse in code are not safe. Please make sure to fix this before deploying.# > S3 bucket definitition = s3.Bucket(name='my.exmple-bucket')# > bucket search prefix = processed/if__name__=='__main__':main()