Repository: justindujardin/pathy (language: python)

Issue:
Azure support
This looks great!

Could you please add support for Azure?


Discussion / hints:

Fix patch:
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 1d1c076..527514e 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -31,6 +31,7 @@ jobs:
           GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }}
           PATHY_S3_ACCESS_ID: ${{ secrets.PATHY_S3_ACCESS_ID }}
           PATHY_S3_ACCESS_SECRET: ${{ secrets.PATHY_S3_ACCESS_SECRET }}
+          PATHY_AZURE_CONNECTION_STRING: ${{ secrets.PATHY_AZURE_CONNECTION_STRING }}
         run: rm -rf ./pathy/ && sh tools/test_package.sh
       - name: Report Code Coverage
         env:
diff --git a/README.md b/README.md
index 119fe00..48ce0bf 100644
--- a/README.md
+++ b/README.md
@@ -41,11 +41,55 @@ assert not greeting.exists()
 
 The table below details the supported cloud provider APIs.
 
-| Cloud Service        | Support |      Install Extras      |
-| :------------------- | :-----: | :----------------------: |
-| Google Cloud Storage |   ✅    | `pip install pathy[gcs]` |
-| Amazon S3            |   ✅    | `pip install pathy[s3]`  |
-| Azure                |   ❌    |                          |
+| Cloud Service        | Support |       Install Extras       |
+| :------------------- | :-----: | :------------------------: |
+| Google Cloud Storage |   ✅    |  `pip install pathy[gcs]`  |
+| Amazon S3            |   ✅    |  `pip install pathy[s3]`   |
+| Azure                |   ✅    | `pip install pathy[azure]` |
+
+### Google Cloud Storage
+
+Google recommends using a JSON credentials file, which you can specify by path:
+
+```python
+from google.oauth2 import service_account
+from pathy import set_client_params
+
+credentials = service_account.Credentials.from_service_account_file("./my-creds.json")
+set_client_params("gs", credentials=credentials)
+```
+
+### Amazon S3
+
+S3 uses a JSON credentials file, which you can specify by path:
+
+```python
+from pathy import set_client_params
+
+set_client_params("s3", key_id="YOUR_ACCESS_KEY_ID", key_secret="YOUR_ACCESS_SECRET")
+```
+
+### Azure
+
+Azure blob storage can be passed a `connection_string`:
+
+```python
+from pathy import set_client_params
+
+set_client_params("azure", connection_string="YOUR_CONNECTION_STRING")
+```
+
+or a `BlobServiceClient` instance:
+
+```python
+from azure.storage.blob import BlobServiceClient
+from pathy import set_client_params
+
+service: BlobServiceClient = BlobServiceClient.from_connection_string(
+    "YOUR_CONNECTION_STRING"
+)
+set_client_params("azure", service=service)
+```
 
 ## Semantic Versioning
 
diff --git a/pathy/__init__.py b/pathy/__init__.py
index 7dd5575..f42a758 100644
--- a/pathy/__init__.py
+++ b/pathy/__init__.py
@@ -196,9 +196,6 @@ def lookup_bucket(self, path: "Pathy") -> Optional[Bucket]:
     def get_bucket(self, path: "Pathy") -> Bucket:
         raise NotImplementedError(SUBCLASS_ERROR)
 
-    def list_buckets(self) -> Generator[Bucket, None, None]:
-        raise NotImplementedError(SUBCLASS_ERROR)
-
     def list_blobs(
         self,
         path: "Pathy",
@@ -209,7 +206,7 @@ def list_blobs(
 
     def scandir(
         self,
-        path: Optional["Pathy"] = None,
+        path: "Pathy",
         prefix: Optional[str] = None,
         delimiter: Optional[str] = None,
     ) -> "PathyScanDir":
@@ -819,7 +816,7 @@ class PathyScanDir(Iterator[Any], ContextManager[Any], ABC):
     def __init__(
         self,
         client: BucketClient,
-        path: Optional[PurePathy] = None,
+        path: PurePathy,
         prefix: Optional[str] = None,
         delimiter: Optional[str] = None,
     ) -> None:
@@ -1009,14 +1006,9 @@ def get_bucket(self, path: PurePathy) -> BucketFS:
             return BucketFS(str(path.root), bucket=bucket_path)
         raise FileNotFoundError(f"Bucket {path.root} does not exist!")
 
-    def list_buckets(self, **kwargs: Dict[str, Any]) -> Generator[BucketFS, None, None]:
-        for f in self.root.glob("*"):
-            if f.is_dir():
-                yield BucketFS(f.name, f)
-
     def scandir(
         self,
-        path: Optional[Pathy] = None,
+        path: Pathy,
         prefix: Optional[str] = None,
         delimiter: Optional[str] = None,
     ) -> PathyScanDir:
@@ -1072,12 +1064,6 @@ class ScanDirFS(PathyScanDir):
     _client: BucketClientFS
 
     def scandir(self) -> Generator[BucketEntry, None, None]:
-        if self._path is None or not self._path.root:
-            for bucket in self._client.list_buckets():
-                yield BucketEntryFS(bucket.name, is_dir=True, raw=None)
-            return
-        assert self._path is not None
-        assert self._path.root is not None
         scan_path = self._client.root / self._path.root
         if isinstance(self._path, BasePath):
             scan_path = (
diff --git a/pathy/azure.py b/pathy/azure.py
new file mode 100644
index 0000000..4b7e88b
--- /dev/null
+++ b/pathy/azure.py
@@ -0,0 +1,244 @@
+import datetime
+from dataclasses import dataclass
+from typing import Any, Dict, Generator, List, Optional, cast
+
+try:
+    import azure
+    from azure.storage.blob import (
+        BlobClient,
+        BlobProperties,
+        BlobServiceClient,
+        ContainerClient,
+    )
+except (ImportError, ModuleNotFoundError):
+    raise ImportError(
+        """You are using the Azure functionality of Pathy without
+    having the required dependencies installed.
+
+    Please try installing them:
+
+        pip install pathy[azure]
+
+    """
+    )
+
+from . import (
+    Blob,
+    Bucket,
+    BucketClient,
+    BucketEntry,
+    PathyScanDir,
+    PurePathy,
+    register_client,
+)
+
+
+def _safe_last_modified(last_modified: Optional[datetime.datetime]) -> int:
+    """Helper to coerce an optional last_modified timestamp into an int"""
+    return int(last_modified.timestamp()) if last_modified else 0
+
+
+class BucketEntryAzure(BucketEntry):
+    bucket: "BucketAzure"
+    raw: Any
+
+
+@dataclass
+class BlobAzure(Blob):
+    client: BlobClient
+    bucket: "BucketAzure"
+
+    def delete(self) -> None:
+        self.client.delete_blob()
+
+    def exists(self) -> bool:
+        return self.client.exists()
+
+
+@dataclass
+class BucketAzure(Bucket):
+    name: str
+    client: BlobServiceClient
+    container: ContainerClient
+
+    def get_blob(self, blob_name: str) -> Optional[BlobAzure]:
+        blob_client = self.client.get_blob_client(container=self.name, blob=blob_name)
+        if not blob_client.exists():
+            return None
+        blob_stat = blob_client.get_blob_properties()
+        size = blob_stat.size
+        return BlobAzure(
+            client=blob_client,
+            bucket=self,
+            owner=None,  # type:ignore
+            name=blob_name,  # type:ignore
+            raw=None,
+            size=size,
+            updated=_safe_last_modified(blob_stat.last_modified),
+        )
+
+    def copy_blob(  # type:ignore[override]
+        self, blob: BlobAzure, target: "BucketAzure", name: str
+    ) -> Optional[BlobAzure]:
+
+        copied_blob = self.client.get_blob_client(target.container.container_name, name)
+        copied_blob.start_copy_from_url(blob.client.url)
+        props = copied_blob.get_blob_properties()
+        if props.copy.status != "success" and props.copy.id is not None:
+            copied_blob.abort_copy(props.copy.id)
+            return None
+        pathy_blob = BlobAzure(
+            client=copied_blob,
+            bucket=target,
+            owner=None,
+            name=copied_blob.blob_name,
+            raw=None,
+            size=props.size,
+            updated=_safe_last_modified(props.last_modified),
+        )
+        return pathy_blob
+
+    def delete_blob(self, blob: BlobAzure) -> None:  # type:ignore[override]
+        blob.delete()
+
+    def delete_blobs(self, blobs: List[BlobAzure]) -> None:  # type:ignore[override]
+        for blob in blobs:
+            self.delete_blob(blob)
+
+    def exists(self) -> bool:
+        return self.container.exists()
+
+
+class BucketClientAzure(BucketClient):
+    _service: BlobServiceClient
+
+    @property
+    def client_params(self) -> Dict[str, Any]:
+        return dict(client=self._service)
+
+    def __init__(self, **kwargs: Any) -> None:
+        self.recreate(**kwargs)
+
+    def recreate(self, **kwargs: Any) -> None:
+        self._service = kwargs.get("service", None)
+        if self._service is None:
+            connection_string = kwargs.get("connection_string", None)
+            if connection_string is None:
+                raise ValueError("Expected either 'service' or 'connection_string'")
+            self._service = BlobServiceClient.from_connection_string(connection_string)
+
+    def make_uri(self, path: PurePathy) -> str:
+        return str(path)
+
+    def create_bucket(  # type:ignore[override]
+        self, path: PurePathy
+    ) -> ContainerClient:
+        container = self._service.get_container_client(container=path.root)
+        container.create_container()
+        return container
+
+    def delete_bucket(self, path: PurePathy) -> None:
+        container = self._service.get_container_client(container=path.root)
+        container.delete_container()
+
+    def exists(self, path: PurePathy) -> bool:
+        # Because we want all the parents of a valid blob (e.g. "directory" in
+        # "directory/foo.file") to return True, we enumerate the blobs with a prefix
+        # and compare the object names to see if they match a substring of the path
+        key_name = str(path.key)
+        for obj in self.list_blobs(path):
+            if obj.name.startswith(key_name + path._flavour.sep):  # type:ignore
+                return True
+        return False
+
+    def lookup_bucket(self, path: PurePathy) -> Optional[BucketAzure]:
+        try:
+            return self.get_bucket(path)
+        except FileNotFoundError:
+            return None
+
+    def get_bucket(self, path: PurePathy) -> BucketAzure:
+        try:
+            native_bucket = self._service.get_container_client(container=path.root)
+            if native_bucket.exists():
+                return BucketAzure(
+                    str(path.root), client=self._service, container=native_bucket
+                )
+        except azure.core.exceptions.HttpResponseError:  # type: ignore
+            pass
+        raise FileNotFoundError(f"Bucket {path.root} does not exist!")
+
+    def scandir(  # type:ignore[override]
+        self,
+        path: PurePathy,
+        prefix: Optional[str] = None,
+        delimiter: Optional[str] = None,
+    ) -> PathyScanDir:
+        return ScanDirAzure(client=self, path=path, prefix=prefix, delimiter=delimiter)
+
+    def list_blobs(
+        self,
+        path: PurePathy,
+        prefix: Optional[str] = None,
+        delimiter: Optional[str] = None,
+    ) -> Generator[BlobAzure, None, None]:
+        bucket = self.lookup_bucket(path)
+        if bucket is None:
+            return
+        paginator = bucket.container.list_blobs(name_starts_with=prefix)
+        props: BlobProperties
+        for props in paginator:
+            prop_name = cast(str, props.name)
+            blob_client = bucket.container.get_blob_client(prop_name)
+            yield BlobAzure(
+                client=blob_client,
+                bucket=bucket,
+                owner=None,
+                name=prop_name,
+                raw=props,
+                size=props.size,
+                updated=_safe_last_modified(props.last_modified),
+            )
+
+
+class ScanDirAzure(PathyScanDir):
+    _client: BucketClientAzure
+
+    def __init__(
+        self,
+        client: BucketClient,
+        path: PurePathy,
+        prefix: Optional[str] = None,
+        delimiter: Optional[str] = None,
+        page_size: Optional[int] = None,
+    ) -> None:
+        super().__init__(client=client, path=path, prefix=prefix, delimiter=delimiter)
+        self._page_size = page_size
+
+    def scandir(self) -> Generator[BucketEntryAzure, None, None]:
+        bucket = self._client.lookup_bucket(self._path)
+        if bucket is None:
+            return
+        paginator = bucket.container.list_blobs(name_starts_with=self._prefix)
+        blob: BlobProperties
+        seen_dirs: Dict[str, bool] = {}
+        for blob in paginator:
+            is_dir = False
+            blob_name = cast(str, blob.name)
+            if self._prefix:
+                blob_name = blob_name[len(self._prefix) :]
+            if blob_name.find("/") != -1:
+                blob_name = blob_name[0 : blob_name.find("/")]
+                if blob_name in seen_dirs:
+                    continue
+                seen_dirs[blob_name] = True
+                is_dir = True
+            yield BucketEntryAzure(
+                name=blob_name,
+                is_dir=is_dir,
+                size=cast(int, blob.size),
+                last_modified=_safe_last_modified(blob.last_modified),
+            )
+
+
+register_client("azure", BucketClientAzure)
diff --git a/pathy/gcs.py b/pathy/gcs.py
index 9c384e4..9bed278 100644
--- a/pathy/gcs.py
+++ b/pathy/gcs.py
@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import Any, Dict, Generator, List, Optional
+from typing import Any, Generator, List, Optional
 
 try:
     from google.api_core.exceptions import BadRequest  # type:ignore
@@ -143,14 +143,9 @@ def get_bucket(self, path: PurePathy) -> BucketGCS:
             pass
         raise FileNotFoundError(f"Bucket {path.root} does not exist!")
 
-    def list_buckets(  # type:ignore[override]
-        self, **kwargs: Dict[str, Any]
-    ) -> Generator[GCSNativeBucket, None, None]:
-        return self.client.list_buckets(**kwargs)  # type:ignore
-
     def scandir(  # type:ignore[override]
         self,
-        path: Optional[PurePathy] = None,
+        path: PurePathy,
         prefix: Optional[str] = None,
         delimiter: Optional[str] = None,
     ) -> PathyScanDir:
@@ -184,13 +179,6 @@ class ScanDirGCS(PathyScanDir):
     _client: BucketClientGCS
 
     def scandir(self) -> Generator[BucketEntryGCS, None, None]:
-        if self._path is None or not self._path.root:
-            gcs_bucket: GCSNativeBucket
-            for gcs_bucket in self._client.list_buckets():
-                yield BucketEntryGCS(
-                    gcs_bucket.name, is_dir=True, raw=None  # type:ignore
-                )
-            return
         sep = self._path._flavour.sep  # type:ignore
         bucket = self._client.lookup_bucket(self._path)
         if bucket is None:
diff --git a/pathy/s3.py b/pathy/s3.py
index 53ef710..767d3cf 100644
--- a/pathy/s3.py
+++ b/pathy/s3.py
@@ -168,16 +168,9 @@ def get_bucket(self, path: PurePathy) -> BucketS3:
         except (ClientError, ParamValidationError):
             raise FileNotFoundError(f"Bucket {path.root} does not exist!")
 
-    def list_buckets(  # type:ignore[override]
-        self, **kwargs: Dict[str, Any]
-    ) -> Generator[S3NativeBucket, None, None]:
-        native_buckets = self.client.list_buckets(**kwargs)["Buckets"]
-        results = (BucketS3(n["Name"], self.client, n) for n in native_buckets)
-        return results
-
     def scandir(  # type:ignore[override]
         self,
-        path: Optional[PurePathy] = None,
+        path: PurePathy,
         prefix: Optional[str] = None,
         delimiter: Optional[str] = None,
     ) -> PathyScanDir:
@@ -215,7 +208,7 @@ class ScanDirS3(PathyScanDir):
     def __init__(
         self,
         client: BucketClient,
-        path: Optional[PurePathy] = None,
+        path: PurePathy,
         prefix: Optional[str] = None,
         delimiter: Optional[str] = None,
         page_size: Optional[int] = None,
@@ -224,11 +217,6 @@ def __init__(
         self._page_size = page_size
 
     def scandir(self) -> Generator[BucketEntryS3, None, None]:
-        if self._path is None or not self._path.root:
-            s3_bucket: BucketS3
-            for s3_bucket in self._client.list_buckets():
-                yield BucketEntryS3(s3_bucket.name, is_dir=True, raw=None)
-            return
         sep = self._path._flavour.sep  # type:ignore
         bucket = self._client.lookup_bucket(self._path)
         if bucket is None:
diff --git a/setup.py b/setup.py
index 4a4ca67..2cebfa4 100644
--- a/setup.py
+++ b/setup.py
@@ -21,6 +21,7 @@ def setup_package():
     extras = {
         "gcs": ["google-cloud-storage>=1.26.0,<2.0.0"],
         "s3": ["boto3"],
+        "azure": ["azure-storage-blob"],
         "test": ["pytest", "pytest-coverage", "mock", "typer-cli"],
     }
     extras["all"] = [item for group in extras.values() for item in group]


Test patch:
diff --git a/pathy/_tests/__init__.py b/pathy/_tests/__init__.py
index 5b9b883..38d1e8f 100644
--- a/pathy/_tests/__init__.py
+++ b/pathy/_tests/__init__.py
@@ -23,3 +23,17 @@
 except ImportError:
     s3_installed = False
     s3_testable = False
+
+
+azure_testable: bool
+azure_installed: bool
+
+
+try:
+    from ..azure import BucketClientAzure
+
+    azure_installed = bool(BucketClientAzure)
+    azure_testable = azure_installed and "PATHY_AZURE_CONNECTION_STRING" in os.environ
+except ImportError:
+    azure_installed = False
+    azure_testable = False
diff --git a/pathy/_tests/conftest.py b/pathy/_tests/conftest.py
index bc4957b..4d03f93 100644
--- a/pathy/_tests/conftest.py
+++ b/pathy/_tests/conftest.py
@@ -1,5 +1,6 @@
 import json
 import os
+import random
 import shutil
 import sys
 import tempfile
@@ -10,7 +11,7 @@
 
 from pathy import Pathy, set_client_params, use_fs, use_fs_cache
 
-from . import gcs_testable, s3_testable
+from . import azure_testable, gcs_testable, s3_testable
 
 # Which adapters to use
 TEST_ADAPTERS = ["fs"]
@@ -18,6 +19,8 @@
     TEST_ADAPTERS.append("gcs")
 if s3_testable:
     TEST_ADAPTERS.append("s3")
+if azure_testable:
+    TEST_ADAPTERS.append("azure")
 
 # A unique identifier used to allow each python version and OS to test
 # with separate bucket paths. This makes it possible to parallelize the
@@ -97,6 +100,17 @@ def s3_credentials_from_env() -> Optional[Tuple[str, str]]:
     return (access_key_id, access_secret)
 
 
+def azure_credentials_from_env() -> Optional[str]:
+    """Extract an access key ID and Secret from the environment."""
+    if not azure_testable:
+        return None
+
+    connection_string: Optional[str] = os.environ.get(
+        "PATHY_AZURE_CONNECTION_STRING", None
+    )
+    return connection_string
+
+
 @pytest.fixture()
 def with_adapter(
     adapter: str, bucket: str, other_bucket: str
@@ -104,7 +118,7 @@ def with_adapter(
     tmp_dir = None
     scheme = "gs"
     if adapter == "gcs":
-        # Use GCS
+        # Use google-cloud-storage
         use_fs(False)
         credentials = gcs_credentials_from_env()
         if credentials is not None:
@@ -117,6 +131,23 @@ def with_adapter(
         if credentials is not None:
             key_id, key_secret = credentials
             set_client_params("s3", key_id=key_id, key_secret=key_secret)
+    elif adapter == "azure":
+        scheme = "azure"
+        # Use azure-storage-blob
+        use_fs(False)
+        connection_string = azure_credentials_from_env()
+        assert connection_string is not None, "expected valid connection_string in env"
+        # Sometimes pass BlobServiceClient, and sometimes a connection string
+        if bool(random.getrandbits(1)):
+            from azure.storage.blob import BlobServiceClient
+
+            service: BlobServiceClient = BlobServiceClient.from_connection_string(
+                connection_string
+            )
+            set_client_params("azure", service=service)
+        else:
+            set_client_params("azure", connection_string=connection_string)
+
     elif adapter == "fs":
         # Use local file-system in a temp folder
         tmp_dir = tempfile.mkdtemp()
diff --git a/pathy/_tests/test_azure.py b/pathy/_tests/test_azure.py
new file mode 100644
index 0000000..1f96b15
--- /dev/null
+++ b/pathy/_tests/test_azure.py
@@ -0,0 +1,113 @@
+from dataclasses import dataclass
+
+import mock
+import pytest
+
+from pathy import Pathy, get_client
+
+from . import azure_testable
+from .conftest import ENV_ID
+
+AZURE_ADAPTER = ["azure"]
+
+
+@pytest.mark.parametrize("adapter", AZURE_ADAPTER)
+@pytest.mark.skipif(not azure_testable, reason="requires azure")
+def test_azure_recreate_expected_args(with_adapter: str) -> None:
+    from pathy.azure import BucketClientAzure
+
+    # Must specify either service, or connection_string
+    with pytest.raises(ValueError):
+        BucketClientAzure()
+
+
+@pytest.mark.parametrize("adapter", AZURE_ADAPTER)
+@pytest.mark.skipif(not azure_testable, reason="requires azure")
+def test_azure_bucket_get_blob_failure_cases(with_adapter: str, bucket: str) -> None:
+    root = Pathy(f"{with_adapter}://{bucket}")
+    client = root.client(root)
+    azure_bucket = client.get_bucket(root)
+    # Returns none if blob contains invalid character
+    assert azure_bucket.get_blob("invlid_bkt_nme_18$%^@57582397?.___asd") is None
+
+
+@pytest.mark.parametrize("adapter", AZURE_ADAPTER)
+@pytest.mark.skipif(not azure_testable, reason="requires azure")
+def test_azure_bucket_client_list_blobs(with_adapter: str, bucket: str) -> None:
+    """Test corner-case in Azure client that isn't easily reachable from Pathy"""
+    from pathy.azure import BucketClientAzure
+
+    client: BucketClientAzure = get_client("azure")
+
+    # Invalid bucket
+    root = Pathy("azure://invalid_h3gE_ds5daEf_Sdf15487t2n4")
+    blobs = list(client.list_blobs(root))
+    assert blobs == []
+
+    # Invalid search prefix
+    root = Pathy(f"azure://{bucket}")
+    blobs = list(client.list_blobs(root, prefix="#@/:|*?%@^@$^@$@#$@#$"))
+    assert blobs == []
+
+
+@dataclass
+class MockAzureBlobClientCopyProperties:
+    status: str
+    id: str
+
+
+@dataclass
+class MockAzureBlobClientProperties:
+    copy: MockAzureBlobClientCopyProperties
+
+
+@dataclass
+class MockAzureBlobClient:
+    aborted = False
+
+    def start_copy_from_url(self, url: str) -> None:
+        pass
+
+    def get_blob_properties(self) -> MockAzureBlobClientProperties:
+        return MockAzureBlobClientProperties(
+            MockAzureBlobClientCopyProperties(status="failed", id="test")
+        )
+
+    def abort_copy(self, copy_id: str) -> None:
+        self.aborted = True
+
+
+@dataclass
+class MockAzureBlobServiceClient:
+    mock_client: MockAzureBlobClient
+
+    def get_blob_client(self, container: str, blob_name: str) -> MockAzureBlobClient:
+        return self.mock_client
+
+
+@pytest.mark.parametrize("adapter", AZURE_ADAPTER)
+@pytest.mark.skipif(not azure_testable, reason="requires azure")
+def test_azure_bucket_copy_blob_aborts_copy_on_failure(
+    with_adapter: str, bucket: str
+) -> None:
+    """Verify abort_copy is called when copy status != 'success'"""
+    root: Pathy = Pathy(f"{with_adapter}://{bucket}/{ENV_ID}/azure_abort_copy")
+
+    # Create a source blob to copy
+    cool_blob = root / "file.txt"
+    cool_blob.write_text("cool")
+    client = root.client(root)
+
+    # Get the bucket client / source blob
+    azure_bucket = client.get_bucket(root)
+    azure_blob = azure_bucket.get_blob(cool_blob.prefix[:-1])
+    assert azure_blob is not None
+
+    # Mock out blob client to catch `abort_copy` call
+    mock_blob_client = MockAzureBlobClient()
+    mock_bucket = MockAzureBlobServiceClient(mock_blob_client)
+    assert mock_blob_client.aborted is False
+    with mock.patch.object(azure_bucket, "client", new=mock_bucket):
+        assert azure_bucket.copy_blob(azure_blob, azure_bucket, "file2.txt") is None
+    # abort_copy was called
+    assert mock_blob_client.aborted is True
diff --git a/pathy/_tests/test_base.py b/pathy/_tests/test_base.py
index 9c3b3b9..4882447 100644
--- a/pathy/_tests/test_base.py
+++ b/pathy/_tests/test_base.py
@@ -171,8 +171,6 @@ def test_client_base_bucket_client_raises_not_implemented() -> None:
         client.is_dir(Pathy("gs://foo"))
     with pytest.raises(NotImplementedError):
         client.get_bucket(Pathy("gs://foo"))
-    with pytest.raises(NotImplementedError):
-        client.list_buckets()
     with pytest.raises(NotImplementedError):
         client.list_blobs(Pathy("gs://foo"))
     with pytest.raises(NotImplementedError):
diff --git a/pathy/_tests/test_file.py b/pathy/_tests/test_file.py
index 302ef73..6707cf8 100644
--- a/pathy/_tests/test_file.py
+++ b/pathy/_tests/test_file.py
@@ -4,15 +4,7 @@
 import mock
 import pytest
 
-from pathy import (
-    BlobFS,
-    BucketClientFS,
-    BucketFS,
-    ClientError,
-    Pathy,
-    ScanDirFS,
-    get_client,
-)
+from pathy import BlobFS, BucketClientFS, BucketFS, ClientError, Pathy, get_client
 
 FS_ADAPTER = ["fs"]
 
@@ -110,13 +102,3 @@ def test_file_bucket_client_fs_list_blobs(with_adapter: str) -> None:
 
     blobs = [b.name for b in client.list_blobs(root, prefix="foo/bar/baz")]
     assert len(blobs) == 1
-
-
-@pytest.mark.parametrize("adapter", FS_ADAPTER)
-def test_file_scandir_list_buckets(
-    with_adapter: str, bucket: str, other_bucket: str
-) -> None:
-    root = Pathy()
-    client = root.client(root)
-    scandir = ScanDirFS(client=client, path=root)
-    assert sorted([s.name for s in scandir]) == sorted([bucket, other_bucket])
diff --git a/pathy/_tests/test_gcs.py b/pathy/_tests/test_gcs.py
index 35020df..6630d7f 100644
--- a/pathy/_tests/test_gcs.py
+++ b/pathy/_tests/test_gcs.py
@@ -43,19 +43,6 @@ def test_gcs_import_error_missing_deps() -> None:
         get_client("gs")
 
 
-@pytest.mark.parametrize("adapter", GCS_ADAPTER)
-@pytest.mark.skipif(not gcs_testable, reason="requires gcs")
-def test_gcs_scandir_list_buckets(
-    with_adapter: str, bucket: str, other_bucket: str
-) -> None:
-    from pathy.gcs import ScanDirGCS
-
-    root = Pathy("gs://foo/bar")
-    client = root.client(root)
-    scandir = ScanDirGCS(client=client, path=Pathy())
-    assert sorted([s.name for s in scandir]) == sorted([bucket, other_bucket])
-
-
 @pytest.mark.parametrize("adapter", GCS_ADAPTER)
 @pytest.mark.skipif(not gcs_testable, reason="requires gcs")
 def test_gcs_scandir_invalid_bucket_name(with_adapter: str) -> None:
diff --git a/pathy/_tests/test_pathy.py b/pathy/_tests/test_pathy.py
index e586c97..000eda5 100644
--- a/pathy/_tests/test_pathy.py
+++ b/pathy/_tests/test_pathy.py
@@ -25,13 +25,13 @@
 
 @pytest.mark.parametrize("adapter", TEST_ADAPTERS)
 def test_pathy_is_path_instance(with_adapter: str) -> None:
-    blob = Pathy("gs://fake/blob")
+    blob = Pathy(f"{with_adapter}://fake/blob")
     assert isinstance(blob, Path)
 
 
 @pytest.mark.parametrize("adapter", TEST_ADAPTERS)
 def test_pathy_fluid(with_adapter: str, bucket: str) -> None:
-    path: FluidPath = Pathy.fluid(f"gs://{bucket}/{ENV_ID}/fake-key")
+    path: FluidPath = Pathy.fluid(f"{with_adapter}://{bucket}/{ENV_ID}/fake-key")
     assert isinstance(path, Pathy)
     path = Pathy.fluid("foo/bar.txt")
     assert isinstance(path, BasePath)
@@ -98,10 +98,10 @@ def test_pathy_stat(with_adapter: str, bucket: str) -> None:
 
 @pytest.mark.parametrize("adapter", TEST_ADAPTERS)
 def test_pathy_resolve(with_adapter: str, bucket: str) -> None:
-    path = Pathy(f"gs://{bucket}/{ENV_ID}/fake-key")
+    path = Pathy(f"{with_adapter}://{bucket}/{ENV_ID}/fake-key")
     assert path.resolve() == path
-    path = Pathy(f"gs://{bucket}/{ENV_ID}/dir/../fake-key")
-    assert path.resolve() == Pathy(f"gs://{bucket}/{ENV_ID}/fake-key")
+    path = Pathy(f"{with_adapter}://{bucket}/{ENV_ID}/dir/../fake-key")
+    assert path.resolve() == Pathy(f"{with_adapter}://{bucket}/{ENV_ID}/fake-key")
 
 
 @pytest.mark.parametrize("adapter", TEST_ADAPTERS)
@@ -116,6 +116,7 @@ def test_pathy_exists(with_adapter: str, bucket: str) -> None:
     assert Pathy(f"{with_adapter}://{invalid_bucket}/foo.blob").exists() is False
     # valid bucket with invalid object
     assert Pathy(f"{with_adapter}://{bucket}/not_found_lol_nice.txt").exists() is False
+    assert Pathy(f"{with_adapter}://{bucket}/@^@^#$%@#$.txt").exists() is False
 
     path = Pathy(f"{with_adapter}://{bucket}/{ENV_ID}/directory/foo.txt")
     path.write_text("---")
@@ -155,6 +156,12 @@ def test_pathy_glob(with_adapter: str, bucket: str) -> None:
         root / "1/dir/file.txt",
     ]
 
+    bad_root = Pathy(f"{with_adapter}://{bucket}-bad-name")
+    assert list(bad_root.glob("*.txt")) == []
+
+    bad_name = root / "@#$%@^@%@#@#$.glob"
+    assert list(bad_name.glob("*.glob")) == []
+
 
 @pytest.mark.parametrize("adapter", TEST_ADAPTERS)
 def test_pathy_unlink_path(with_adapter: str, bucket: str) -> None:
@@ -199,9 +206,9 @@ def test_pathy_iterdir(with_adapter: str, bucket: str) -> None:
         path = root / f"{i}.file"
         path.write_text("---")
 
-    # 1 file in a subfolder
-    path = root / "sub/file.txt"
-    path.write_text("---")
+    # 2 files in a subfolder
+    (root / "sub/file.txt").write_text("---")
+    (root / "sub/file2.txt").write_text("---")
 
     check = sorted(root.iterdir())
     assert check == [
diff --git a/pathy/_tests/test_s3.py b/pathy/_tests/test_s3.py
index ef7f4de..14d523a 100644
--- a/pathy/_tests/test_s3.py
+++ b/pathy/_tests/test_s3.py
@@ -8,21 +8,6 @@
 S3_ADAPTER = ["s3"]
 
 
-@pytest.mark.parametrize("adapter", S3_ADAPTER)
-@pytest.mark.skipif(not s3_testable, reason="requires s3")
-def test_s3_scandir_list_buckets(
-    with_adapter: str, bucket: str, other_bucket: str
-) -> None:
-    from pathy.s3 import ScanDirS3
-
-    root = Pathy("s3://foo/bar")
-    client = root.client(root)
-    scandir = ScanDirS3(client=client, path=Pathy())
-    buckets = [s.name for s in scandir]
-    assert bucket in buckets
-    assert other_bucket in buckets
-
-
 @pytest.mark.parametrize("adapter", S3_ADAPTER)
 @pytest.mark.skipif(not s3_testable, reason="requires s3")
 def test_s3_scandir_scandir_continuation_token(