Repository: drivendataorg/cloudpathlib (2020-11-06 ~ 2022-05-15)

problem_statement: Ensure re-instantiation is idempotent
One of my favorite things about `Path` is that it is idempotent, and that means that I can easily handle string or `PathLike` input by just doing `p = Path(p)` at top of my function. This seems to work in the basic case:

```
In [1]: from cloudpathlib import CloudPath

In [2]: c = CloudPath('s3://test/test.txt')

In [3]: CloudPath(c)
Out[3]: S3Path('s3://test/test.txt')
```

We should make sure that `CloudPath` does the same (maybe it already does?) by at least adding some tests. I'm not sure it is truly idempotent and will use the same client if a new one is not passed, etc., etc.
date: 2020-11-06
patch: diff --git a/HISTORY.md b/HISTORY.md
index a5ebb93..fc91c86 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -1,5 +1,9 @@
 # History
 
+## v0.1.2 (Unreleased)
+
+- Fixed `CloudPath` instantiation so that reinstantiating with an existing `CloudPath` instance will reuse the same client, if a new client is not explicitly passed. This addresses the edge case of non-idempotency when reinstantiating a `CloudPath` instance with a non-default client. ([#104](https://github.com/drivendataorg/cloudpathlib/pull/104))
+
 ## v0.1.1 (2020-10-15)
 
 - Fixed a character-encoding bug when building from source on Windows. ([#98](https://github.com/drivendataorg/cloudpathlib/pull/98))
diff --git a/cloudpathlib/azure/azblobpath.py b/cloudpathlib/azure/azblobpath.py
index f2ba599..5b4b034 100644
--- a/cloudpathlib/azure/azblobpath.py
+++ b/cloudpathlib/azure/azblobpath.py
@@ -1,13 +1,19 @@
 import os
 from pathlib import Path
 from tempfile import TemporaryDirectory
+from typing import TYPE_CHECKING
 
 from ..cloudpath import CloudPath, register_path_class
 
 
+if TYPE_CHECKING:
+    from .azblobclient import AzureBlobClient
+
+
 @register_path_class("azure")
 class AzureBlobPath(CloudPath):
     cloud_prefix: str = "az://"
+    client: "AzureBlobClient"
 
     @property
     def drive(self) -> str:
diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py
index 09ad1ac..d638471 100644
--- a/cloudpathlib/cloudpath.py
+++ b/cloudpathlib/cloudpath.py
@@ -4,11 +4,15 @@ import collections.abc
 import fnmatch
 import os
 from pathlib import Path, PosixPath, PurePosixPath, WindowsPath
-from typing import Any, IO, Iterable, Union
+from typing import Any, IO, Iterable, Optional, TYPE_CHECKING, Union
 from urllib.parse import urlparse
 from warnings import warn
 
 
+if TYPE_CHECKING:
+    from .client import Client
+
+
 # Custom Exceptions
 class MissingDependencies(Exception):
     pass
@@ -141,7 +145,7 @@ class CloudPath(metaclass=CloudPathMeta):
     _cloud_meta: CloudImplementation
     cloud_prefix: str
 
-    def __init__(self, cloud_path: Union[str, "CloudPath"], client=None):
+    def __init__(self, cloud_path: Union[str, "CloudPath"], client: Optional["Client"] = None):
         self.is_valid_cloudpath(cloud_path, raise_on_error=True)
 
         # versions of the raw string that provide useful methods
@@ -151,14 +155,17 @@ class CloudPath(metaclass=CloudPathMeta):
 
         # setup client
         if client is None:
-            client = self._cloud_meta.client_class.get_default_client()
-        if type(client) != self._cloud_meta.client_class:
+            if isinstance(cloud_path, CloudPath):
+                client = cloud_path.client
+            else:
+                client = self._cloud_meta.client_class.get_default_client()
+        if not isinstance(client, self._cloud_meta.client_class):
             raise ClientMismatch(
                 f"Client of type [{client.__class__}] is not valid for cloud path of type "
                 f"[{self.__class__}]; must be instance of [{self._cloud_meta.client_class}], or "
                 f"None to use default client for this cloud path class."
             )
-        self.client = client
+        self.client: Client = client
 
         # track if local has been written to, if so it may need to be uploaded
         self._dirty = False
diff --git a/cloudpathlib/s3/s3path.py b/cloudpathlib/s3/s3path.py
index 8290474..52ce226 100644
--- a/cloudpathlib/s3/s3path.py
+++ b/cloudpathlib/s3/s3path.py
@@ -1,13 +1,19 @@
 import os
 from pathlib import Path
 from tempfile import TemporaryDirectory
+from typing import TYPE_CHECKING
 
 from ..cloudpath import CloudPath, register_path_class
 
 
+if TYPE_CHECKING:
+    from .s3client import S3Client
+
+
 @register_path_class("s3")
 class S3Path(CloudPath):
     cloud_prefix: str = "s3://"
+    client: "S3Client"
 
     @property
     def drive(self) -> str:

test_patch: diff --git a/tests/test_cloudpath_instantiation.py b/tests/test_cloudpath_instantiation.py
index f957e6e..48a4886 100644
--- a/tests/test_cloudpath_instantiation.py
+++ b/tests/test_cloudpath_instantiation.py
@@ -56,6 +56,17 @@ def test_instantiation_errors(rig):
         rig.path_class("NOT_S3_PATH")
 
 
+def test_idempotency(rig):
+    rig.client_class._default_client = None
+
+    client = rig.client_class()
+    p = client.CloudPath(f"{rig.cloud_prefix}{rig.drive}/{rig.test_dir}/dir_0/file0_0.txt")
+
+    p2 = CloudPath(p)
+    assert p == p2
+    assert p.client == p2.client
+
+
 def test_dependencies_not_loaded(rig, monkeypatch):
     monkeypatch.setattr(rig.path_class._cloud_meta, "dependencies_loaded", False)
     with pytest.raises(MissingDependencies):


----

problem_statement: Calling `.suffix` on a dir raises IndexError
`Path("/dir").suffix` returns the empty string, and we should do the same.


`S3Path("s3://bucket/dir").suffix` results in:

```python
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-21-161bf2e9d1ca> in <module>
----> 1 S3Path("s3://bucket/dir").suffix

~/miniconda3/envs/zamba-algorithms/lib/python3.6/site-packages/cloudpathlib/cloudpath.py in suffix(self)
    522     @property
    523     def suffix(self):
--> 524         return self._dispatch_to_path("suffix")
    525 
    526     @property

~/miniconda3/envs/zamba-algorithms/lib/python3.6/site-packages/cloudpathlib/cloudpath.py in _dispatch_to_path(self, func, *args, **kwargs)
    470 
    471         if isinstance(path_version, collections.abc.Sequence) and isinstance(
--> 472             path_version[0], PurePosixPath
    473         ):
    474             return [

IndexError: string index out of range



```
date: 2022-02-06
patch: diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py
index d638471..04020b3 100644
--- a/cloudpathlib/cloudpath.py
+++ b/cloudpathlib/cloudpath.py
@@ -3,7 +3,7 @@ from collections import defaultdict
 import collections.abc
 import fnmatch
 import os
-from pathlib import Path, PosixPath, PurePosixPath, WindowsPath
+from pathlib import Path, PosixPath, PurePosixPath, WindowsPath, _PathParents  # type: ignore
 from typing import Any, IO, Iterable, Optional, TYPE_CHECKING, Union
 from urllib.parse import urlparse
 from warnings import warn
@@ -460,14 +460,21 @@ class CloudPath(metaclass=CloudPathMeta):
             path_version = _resolve(path_version)
             return self._new_cloudpath(path_version)
 
-        if isinstance(path_version, collections.abc.Iterable) and isinstance(
-            path_version[0], PurePosixPath
+        # When sequence of PurePosixPath, we want to convert to sequence of CloudPaths
+        if (
+            isinstance(path_version, collections.abc.Sequence)
+            and len(path_version) > 0
+            and isinstance(path_version[0], PurePosixPath)
         ):
-            return [
+            sequence_class = (
+                type(path_version) if not isinstance(path_version, _PathParents) else tuple
+            )
+            return sequence_class(
                 self._new_cloudpath(_resolve(p)) for p in path_version if _resolve(p) != p.root
-            ]
+            )
 
         # when pathlib returns a string, etc. we probably just want that thing
+        # cases this should include: str, empty sequence, sequence of str, ...
         else:
             return path_version
 

test_patch: diff --git a/tests/test_cloudpath_manipulation.py b/tests/test_cloudpath_manipulation.py
index bfec276..239a536 100644
--- a/tests/test_cloudpath_manipulation.py
+++ b/tests/test_cloudpath_manipulation.py
@@ -1,14 +1,26 @@
-def test_joins(rig):
+def test_properties(rig):
     assert rig.create_cloud_path("a/b/c/d").name == "d"
     assert rig.create_cloud_path("a/b/c/d.file").name == "d.file"
+
+    assert rig.create_cloud_path("a/b/c/d").stem == "d"
     assert rig.create_cloud_path("a/b/c/d.file").stem == "d"
+
+    assert rig.create_cloud_path("a/b/c/d").suffix == ""
     assert rig.create_cloud_path("a/b/c/d.file").suffix == ".file"
+
+    assert rig.create_cloud_path("a/b/c/d").suffixes == []
+    assert rig.create_cloud_path("a/b/c/d.tar").suffixes == [".tar"]
     assert rig.create_cloud_path("a/b/c/d.tar.gz").suffixes == [".tar", ".gz"]
+
+
+def test_with_suffix(rig):
     assert (
         str(rig.create_cloud_path("a/b/c/d.file").with_suffix(".png"))
         == f"{rig.cloud_prefix}{rig.drive}/{rig.test_dir}/a/b/c/d.png"
     )
 
+
+def test_joins(rig):
     assert rig.create_cloud_path("a") / "b" == rig.create_cloud_path("a/b")
     assert rig.create_cloud_path("a/b/c/d") / "../../b" == rig.create_cloud_path("a/b/b")
 
@@ -19,13 +31,13 @@ def test_joins(rig):
     assert rig.create_cloud_path("a/b/c/d").anchor == rig.cloud_prefix
     assert rig.create_cloud_path("a/b/c/d").parent == rig.create_cloud_path("a/b/c")
 
-    assert rig.create_cloud_path("a/b/c/d").parents == [
+    assert rig.create_cloud_path("a/b/c/d").parents == (
         rig.create_cloud_path("a/b/c"),
         rig.create_cloud_path("a/b"),
         rig.create_cloud_path("a"),
         rig.path_class(f"{rig.cloud_prefix}{rig.drive}/{rig.test_dir}"),
         rig.path_class(f"{rig.cloud_prefix}{rig.drive}"),
-    ]
+    )
 
     assert rig.create_cloud_path("a").joinpath("b", "c") == rig.create_cloud_path("a/b/c")
 


----

problem_statement: Cannot pickle path objects
```python
Python 3.9.12 (main, Apr  5 2022, 06:56:58) 
[GCC 7.5.0] :: Anaconda, Inc. on linux
>>> import pickle
>>> pickle.format_version
'4.0'
>>> import cloudpathlib
>>> cloudpathlib.__version__
'0.7.1'
>>> with open("/home/matthew/s3.pkl", "w") as f:
...     pickle.dump(cloudpathlib.S3Path("s3://bucket/key"), f)
... 
Traceback (most recent call last):
  File "<stdin>", line 2, in <module>
AttributeError: Can't pickle local object 'lazy_call.<locals>._handler'
```

`pickle` docs: [What can be pickled and unpickled?](https://docs.python.org/3/library/pickle.html#what-can-be-pickled-and-unpickled)
date: 2022-05-15
patch: diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py
index 04020b3..bcff516 100644
--- a/cloudpathlib/cloudpath.py
+++ b/cloudpathlib/cloudpath.py
@@ -178,6 +178,19 @@ class CloudPath(metaclass=CloudPathMeta):
         if self._handle is not None:
             self._handle.close()
 
+    def __getstate__(self):
+        state = self.__dict__.copy()
+
+        # don't pickle client
+        del state["client"]
+
+        return state
+
+    def __setstate__(self, state):
+        client = self._cloud_meta.client_class.get_default_client()
+        state["client"] = client
+        return self.__dict__.update(state)
+
     @property
     def _no_prefix(self) -> str:
         return self._str[len(self.cloud_prefix) :]

test_patch: diff --git a/tests/test_cloudpath_file_io.py b/tests/test_cloudpath_file_io.py
index 71a460c..1d32c8a 100644
--- a/tests/test_cloudpath_file_io.py
+++ b/tests/test_cloudpath_file_io.py
@@ -1,6 +1,7 @@
 from datetime import datetime
 import os
 from pathlib import PurePosixPath
+import pickle
 from time import sleep
 
 import pytest
@@ -99,3 +100,21 @@ def test_os_open(rig):
     p = rig.create_cloud_path("dir_0/file0_0.txt")
     with open(p, "r") as f:
         assert f.readable()
+
+
+def test_pickle(rig, tmpdir):
+    p = rig.create_cloud_path("dir_0/file0_0.txt")
+
+    with (tmpdir / "test.pkl").open("wb") as f:
+        pickle.dump(p, f)
+
+    with (tmpdir / "test.pkl").open("rb") as f:
+        pickled = pickle.load(f)
+
+    # test a call to the network
+    assert pickled.exists()
+
+    # check we unpickled, and that client is the default client
+    assert str(pickled) == str(p)
+    assert pickled.client == p.client
+    assert rig.client_class._default_client == pickled.client