Repository: great-expectations/great_expectations

Issue:
Add service principal authentication support for `TupleAzureBlobStoreBackend`
Currently, `TupleAzureBlobStoreBackend` only supports blob connection string (`AZURE_STORAGE_CONNECTION_STRING`). However, in many companies, the preferred way to authenticate is through service accounts ("service principals" as Msoft calls them).

Could possibly be done using [adlfs](https://github.com/dask/adlfs):
```python
from adlfs import AzureBlobFileSystem

fs = AzureBlobFileSystem(
    account_name=storage_account_name,
    tenant_id=tenant_id,
    client_id=client_id,
    client_secret=client_secret,
)
fs.cp(from_path, to_path)
fs.exists(path)
fs.download(rpath, lpath)
...
```



Discussion:
Hi @Trymzet - thank you so much for submitting this issue! After some internal discussion, this is something that we can't prioritize in the near future. That said, if this is something that you would like to contribute, I'd be happy to offer support and guidance. Is this something in which you might be interested?
We may also be doing some work on authentication for an Azure DataConnector which could dovetail nicely with this. If that is the case, someone working on this from the core team will be in touch.

Resolving patch:
diff --git a/great_expectations/data_context/store/tuple_store_backend.py b/great_expectations/data_context/store/tuple_store_backend.py
--- a/great_expectations/data_context/store/tuple_store_backend.py
+++ b/great_expectations/data_context/store/tuple_store_backend.py
@@ -1,4 +1,5 @@
 # PYTHON 2 - py2 - update to ABC direct use rather than __metaclass__ once we drop py2 support
+import functools
 import logging
 import os
 import random
@@ -970,8 +971,9 @@ class TupleAzureBlobStoreBackend(TupleStoreBackend):
     def __init__(
         self,
         container,
-        connection_string,
-        prefix="",
+        connection_string=None,
+        account_url=None,
+        prefix=None,
         filepath_template=None,
         filepath_prefix=None,
         filepath_suffix=None,
@@ -993,30 +995,39 @@ def __init__(
             manually_initialize_store_backend_id=manually_initialize_store_backend_id,
             store_name=store_name,
         )
-        self.connection_string = connection_string
-        self.prefix = prefix
+        self.connection_string = connection_string or os.environ.get(
+            "AZURE_STORAGE_CONNECTION_STRING"
+        )
+        self.prefix = prefix or ""
         self.container = container
+        self.account_url = account_url or os.environ.get("AZURE_STORAGE_ACCOUNT_URL")
 
-    def _get_container_client(self):
+    @property
+    @functools.lru_cache()
+    def _container_client(self):
 
+        from azure.identity import DefaultAzureCredential
         from azure.storage.blob import BlobServiceClient
 
         if self.connection_string:
-            return BlobServiceClient.from_connection_string(
+            blob_service_client = BlobServiceClient.from_connection_string(
                 self.connection_string
-            ).get_container_client(self.container)
+            )
+        elif self.account_url:
+            blob_service_client = BlobServiceClient(
+                account_url=self.account_url, credential=DefaultAzureCredential()
+            )
         else:
             raise StoreBackendError(
                 "Unable to initialize ServiceClient, AZURE_STORAGE_CONNECTION_STRING should be set"
             )
 
+        return blob_service_client.get_container_client(self.container)
+
     def _get(self, key):
         az_blob_key = os.path.join(self.prefix, self._convert_key_to_filepath(key))
         return (
-            self._get_container_client()
-            .download_blob(az_blob_key)
-            .readall()
-            .decode("utf-8")
+            self._container_client.download_blob(az_blob_key).readall().decode("utf-8")
         )
 
     def _set(self, key, value, content_encoding="utf-8", **kwargs):
@@ -1028,7 +1039,7 @@ def _set(self, key, value, content_encoding="utf-8", **kwargs):
         if isinstance(value, str):
             if az_blob_key.endswith(".html"):
                 my_content_settings = ContentSettings(content_type="text/html")
-                self._get_container_client().upload_blob(
+                self._container_client.upload_blob(
                     name=az_blob_key,
                     data=value,
                     encoding=content_encoding,
@@ -1036,14 +1047,14 @@ def _set(self, key, value, content_encoding="utf-8", **kwargs):
                     content_settings=my_content_settings,
                 )
             else:
-                self._get_container_client().upload_blob(
+                self._container_client.upload_blob(
                     name=az_blob_key,
                     data=value,
                     encoding=content_encoding,
                     overwrite=True,
                 )
         else:
-            self._get_container_client().upload_blob(
+            self._container_client.upload_blob(
                 name=az_blob_key, data=value, overwrite=True
             )
         return az_blob_key
@@ -1052,9 +1063,7 @@ def list_keys(self, prefix: Tuple = ()) -> List[Tuple]:
         # Note that the prefix arg is only included to maintain consistency with the parent class signature
         key_list = []
 
-        for obj in self._get_container_client().list_blobs(
-            name_starts_with=self.prefix
-        ):
+        for obj in self._container_client.list_blobs(name_starts_with=self.prefix):
             az_blob_key = os.path.relpath(obj.name)
             if az_blob_key.startswith(f"{self.prefix}/"):
                 az_blob_key = az_blob_key[len(self.prefix) + 1 :]
@@ -1076,7 +1085,7 @@ def get_url_for_key(self, key, protocol=None):
         az_blob_path = os.path.join(self.container, self.prefix, az_blob_key)
 
         return "https://{}.blob.core.windows.net/{}".format(
-            self._get_container_client().account_name,
+            self._container_client.account_name,
             az_blob_path,
         )
 
@@ -1093,8 +1102,8 @@ def _move(self, source_key, dest_key, **kwargs) -> None:
             dest_blob_path = os.path.join(self.prefix, dest_blob_path)
 
         # azure storage sdk does not have _move method
-        source_blob = self._get_container_client().get_blob_client(source_blob_path)
-        dest_blob = self._get_container_client().get_blob_client(dest_blob_path)
+        source_blob = self._container_client.get_blob_client(source_blob_path)
+        dest_blob = self._container_client.get_blob_client(dest_blob_path)
 
         dest_blob.start_copy_from_url(source_blob.url, requires_sync=True)
         copy_properties = dest_blob.get_blob_properties().copy
@@ -1114,7 +1123,7 @@ def remove_key(self, key):
         if not az_blob_path.startswith(self.prefix):
             az_blob_path = os.path.join(self.prefix, az_blob_path)
 
-        blob = self._get_container_client().get_blob_client(az_blob_path)
+        blob = self._container_client.get_blob_client(az_blob_path)
         blob.delete_blob()
         return True
 


Test patch:
diff --git a/tests/data_context/store/test_store_backends.py b/tests/data_context/store/test_store_backends.py
--- a/tests/data_context/store/test_store_backends.py
+++ b/tests/data_context/store/test_store_backends.py
@@ -2,7 +2,7 @@
 import json
 import os
 from collections import OrderedDict
-from unittest.mock import patch
+from unittest.mock import MagicMock, Mock, patch
 
 import boto3
 import pyparsing as pp
@@ -1191,14 +1191,15 @@ def test_TupleGCSStoreBackend():
     )
 
 
-def test_TupleAzureBlobStoreBackend():
-    pytest.importorskip("azure-storage-blob")
+def test_TupleAzureBlobStoreBackend_connection_string():
+    pytest.importorskip("azure.storage.blob")
+    pytest.importorskip("azure.identity")
     """
     What does this test test and why?
     Since no package like moto exists for Azure-Blob services, we mock the Azure-blob client
     and assert that the store backend makes the right calls for set, get, and list.
     """
-    connection_string = "this_is_a_test_conn_string"
+    connection_string = "DefaultEndpointsProtocol=https;AccountName=dummy;AccountKey=secret;EndpointSuffix=core.windows.net"
     prefix = "this_is_a_test_prefix"
     container = "dummy-container"
 
@@ -1210,53 +1211,58 @@ def test_TupleAzureBlobStoreBackend():
         "azure.storage.blob.BlobServiceClient", autospec=True
     ) as mock_azure_blob_client:
 
-        mock_container_client = mock_azure_blob_client.get_container_client.return_value
+        mock_container_client = my_store._container_client
+        mock_azure_blob_client.from_connection_string.assert_called_once()
 
         my_store.set(("AAA",), "aaa")
-
-        mock_azure_blob_client.from_connection_string.assert_called_once()
-        mock_container_client.assert_called_once()
         mock_container_client.upload_blob.assert_called_once_with(
-            name="AAA", data=b"aaa", encoding="utf-8"
+            name="this_is_a_test_prefix/AAA",
+            data="aaa",
+            encoding="utf-8",
+            overwrite=True,
         )
 
-    with patch(
-        "azure.storage.blob.BlobServiceClient", autospec=True
-    ) as mock_azure_blob_client:
-
-        mock_container_client = mock_azure_blob_client.get_container_client.return_value
-
-        my_store.set(("BBB",), b"bbb")
-
-        mock_azure_blob_client.from_connection_string.assert_called_once()
-        mock_container_client.assert_called_once()
-        mock_container_client.upload_blob.assert_called_once_with(
-            name="AAA", data=b"aaa"
+        my_store.get(("BBB",))
+        mock_container_client.download_blob.assert_called_once_with(
+            "this_is_a_test_prefix/BBB"
         )
 
-    with patch(
-        "azure.storage.blob.BlobServiceClient", autospec=True
-    ) as mock_azure_blob_client:
+        my_store.list_keys()
+        mock_container_client.list_blobs.assert_called_once_with(
+            name_starts_with="this_is_a_test_prefix"
+        )
 
-        mock_container_client = mock_azure_blob_client.get_container_client.return_value
 
-        my_store.get(("BBB",))
+def test_TupleAzureBlobStoreBackend_account_url():
+    pytest.importorskip("azure.storage.blob")
+    pytest.importorskip("azure.identity")
+    """
+    What does this test test and why?
+    Since no package like moto exists for Azure-Blob services, we mock the Azure-blob client
+    and assert that the store backend makes the right calls for set, get, and list.
+    """
+    account_url = "this_is_a_test_account_url"
+    prefix = "this_is_a_test_prefix"
+    container = "dummy-container"
 
-        mock_azure_blob_client.from_connection_string.assert_called_once()
-        mock_container_client.assert_called_once()
-        mock_container_client.download_blob.assert_called_once_with("BBB")
+    my_store = TupleAzureBlobStoreBackend(
+        account_url=account_url, prefix=prefix, container=container
+    )
 
     with patch(
         "azure.storage.blob.BlobServiceClient", autospec=True
     ) as mock_azure_blob_client:
-
-        mock_container_client = mock_azure_blob_client.get_container_client.return_value
-
-        my_store.list_keys()
-
-        mock_azure_blob_client.from_connection_string.assert_called_once()
-        mock_container_client.assert_called_once()
-        mock_container_client.list_blobs.assert_called_once_with("this_is_a_prefix")
+        with patch(
+            "azure.identity.DefaultAzureCredential", autospec=True
+        ) as mock_azure_credential:
+            mock_container_client = my_store._container_client
+            mock_azure_blob_client.assert_called_once()
+
+            my_store.get(("BBB",))
+            mock_container_client.download_blob.assert_called_once_with(
+                "this_is_a_test_prefix/BBB"
+            )
+            mock_azure_credential.assert_called_once()
 
 
 @mock_s3