Repository: apache/airflow

Issue:
Airflow doesn't re-use a secrets backend instance when loading configuration values
### Apache Airflow version

main (development)

### What happened

When airflow is loading its configuration, it creates a new secrets backend instance for each configuration backend it loads from secrets and then additionally creates a global secrets backend instance that is used in `ensure_secrets_loaded` which code outside of the configuration file uses. This can cause issues with the vault backend (and possibly others, not sure) since logging in to vault can be an expensive operation server-side and each instance of the vault secrets backend needs to re-login to use its internal client.

### What you think should happen instead

Ideally, airflow would attempt to create a single secrets backend instance and re-use this. This can possibly be patched in the vault secrets backend, but instead I think updating the `configuration` module to cache the secrets backend would be preferable since it would then apply to any secrets backend.

### How to reproduce

Use the hashicorp vault secrets backend and store some configuration in `X_secret` values. See that it logs in more than you'd expect.

### Operating System

Ubuntu 18.04

### Versions of Apache Airflow Providers

```
apache-airflow==2.3.0
apache-airflow-providers-hashicorp==2.2.0
hvac==0.11.2
```

### Deployment

Official Apache Airflow Helm Chart

### Deployment details

_No response_

### Anything else

_No response_

### Are you willing to submit PR?

- [X] Yes I am willing to submit a PR!

### Code of Conduct

- [X] I agree to follow this project's [Code of Conduct](https://github.com/apache/airflow/blob/main/CODE_OF_CONDUCT.md)



Discussion:

Resolving patch:
diff --git a/airflow/configuration.py b/airflow/configuration.py
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -1534,7 +1534,6 @@ def ensure_secrets_loaded() -> List[BaseSecretsBackend]:
 def get_custom_secret_backend() -> Optional[BaseSecretsBackend]:
     """Get Secret Backend if defined in airflow.cfg"""
     secrets_backend_cls = conf.getimport(section='secrets', key='backend')
-
     if secrets_backend_cls:
         try:
             backends: Any = conf.get(section='secrets', key='backend_kwargs', fallback='{}')
@@ -1542,10 +1541,16 @@ def get_custom_secret_backend() -> Optional[BaseSecretsBackend]:
         except JSONDecodeError:
             alternative_secrets_config_dict = {}
 
-        return secrets_backend_cls(**alternative_secrets_config_dict)
+        return _custom_secrets_backend(secrets_backend_cls, **alternative_secrets_config_dict)
     return None
 
 
+@functools.lru_cache(maxsize=2)
+def _custom_secrets_backend(secrets_backend_cls, **alternative_secrets_config_dict):
+    """Separate function to create secrets backend instance to allow caching"""
+    return secrets_backend_cls(**alternative_secrets_config_dict)
+
+
 def initialize_secrets_backends() -> List[BaseSecretsBackend]:
     """
     * import secrets backend classes


Test patch:
diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py
--- a/tests/core/test_configuration.py
+++ b/tests/core/test_configuration.py
@@ -33,6 +33,7 @@
 from airflow.configuration import (
     AirflowConfigException,
     AirflowConfigParser,
+    _custom_secrets_backend,
     conf,
     expand_env_var,
     get_airflow_config,
@@ -296,6 +297,7 @@ def test_hidding_of_sensitive_config_values(self):
     def test_config_raise_exception_from_secret_backend_connection_error(self, mock_hvac):
         """Get Config Value from a Secret Backend"""
 
+        _custom_secrets_backend.cache_clear()
         mock_client = mock.MagicMock()
         # mock_client.side_effect = AirflowConfigException
         mock_hvac.Client.return_value = mock_client
@@ -322,6 +324,7 @@ def test_config_raise_exception_from_secret_backend_connection_error(self, mock_
             ),
         ):
             test_conf.get('test', 'sql_alchemy_conn')
+        _custom_secrets_backend.cache_clear()
 
     def test_getboolean(self):
         """Test AirflowConfigParser.getboolean"""
@@ -1297,3 +1300,81 @@ def test_conf_as_dict_when_deprecated_value_in_secrets_disabled_config(
                 conf.read_dict(dictionary=cfg_dict)
                 os.environ.clear()
                 assert conf.get('database', 'sql_alchemy_conn') == f'sqlite:///{HOME_DIR}/airflow/airflow.db'
+
+    @mock.patch("airflow.providers.hashicorp._internal_client.vault_client.hvac")
+    @conf_vars(
+        {
+            ("secrets", "backend"): "airflow.providers.hashicorp.secrets.vault.VaultBackend",
+            ("secrets", "backend_kwargs"): '{"url": "http://127.0.0.1:8200", "token": "token"}',
+        }
+    )
+    def test_config_from_secret_backend_caches_instance(self, mock_hvac):
+        """Get Config Value from a Secret Backend"""
+        _custom_secrets_backend.cache_clear()
+
+        test_config = '''[test]
+sql_alchemy_conn_secret = sql_alchemy_conn
+secret_key_secret = secret_key
+'''
+        test_config_default = '''[test]
+sql_alchemy_conn = airflow
+secret_key = airflow
+'''
+
+        mock_client = mock.MagicMock()
+        mock_hvac.Client.return_value = mock_client
+
+        def fake_read_secret(path, mount_point, version):
+            if path.endswith('sql_alchemy_conn'):
+                return {
+                    'request_id': '2d48a2ad-6bcb-e5b6-429d-da35fdf31f56',
+                    'lease_id': '',
+                    'renewable': False,
+                    'lease_duration': 0,
+                    'data': {
+                        'data': {'value': 'fake_conn'},
+                        'metadata': {
+                            'created_time': '2020-03-28T02:10:54.301784Z',
+                            'deletion_time': '',
+                            'destroyed': False,
+                            'version': 1,
+                        },
+                    },
+                    'wrap_info': None,
+                    'warnings': None,
+                    'auth': None,
+                }
+            if path.endswith('secret_key'):
+                return {
+                    'request_id': '2d48a2ad-6bcb-e5b6-429d-da35fdf31f56',
+                    'lease_id': '',
+                    'renewable': False,
+                    'lease_duration': 0,
+                    'data': {
+                        'data': {'value': 'fake_key'},
+                        'metadata': {
+                            'created_time': '2020-03-28T02:10:54.301784Z',
+                            'deletion_time': '',
+                            'destroyed': False,
+                            'version': 1,
+                        },
+                    },
+                    'wrap_info': None,
+                    'warnings': None,
+                    'auth': None,
+                }
+
+        mock_client.secrets.kv.v2.read_secret_version.side_effect = fake_read_secret
+
+        test_conf = AirflowConfigParser(default_config=parameterized_config(test_config_default))
+        test_conf.read_string(test_config)
+        test_conf.sensitive_config_values = test_conf.sensitive_config_values | {
+            ('test', 'sql_alchemy_conn'),
+            ('test', 'secret_key'),
+        }
+
+        assert 'fake_conn' == test_conf.get('test', 'sql_alchemy_conn')
+        mock_hvac.Client.assert_called_once()
+        assert 'fake_key' == test_conf.get('test', 'secret_key')
+        mock_hvac.Client.assert_called_once()
+        _custom_secrets_backend.cache_clear()