Repository: drivendataorg/cloudpathlib
Pull request #403 (2024-02-17 21:14:03)

Stream to and from Azure
Stream to and from Azure. Based on #334.


github-actions[bot]: <!-- NETLIFY DEPLOY COMMENT GENERATED BY ACTIONS_NETLIFY - APP ID SHA256: 9d0998be292ddace9530fffad95434f4e2279530e25cf278a25f07954faa426e -->
🚀 Deployed on https://deploy-preview-403--gallant-agnesi-5f7bb3.netlify.app
codecov[bot]: ## [Codecov](https://app.codecov.io/gh/drivendataorg/cloudpathlib/pull/403?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=drivendataorg) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Comparison is base [(`a242f68`)](https://app.codecov.io/gh/drivendataorg/cloudpathlib/commit/a242f68d762cdf6a7cfc56e5962a8d83bced5f1b?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=drivendataorg) 94.3% compared to head [(`071a67d`)](https://app.codecov.io/gh/drivendataorg/cloudpathlib/pull/403?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=drivendataorg) 93.3%.

> :exclamation: Current head 071a67d differs from pull request most recent head e9899fb. Consider uploading reports for the commit e9899fb to get more accurate results

<details><summary>Additional details and impacted files</summary>


```diff
@@           Coverage Diff            @@
##           master    #403     +/-   ##
========================================
- Coverage    94.3%   93.3%   -1.1%     
========================================
  Files          22      22             
  Lines        1600    1609      +9     
========================================
- Hits         1510    1502      -8     
- Misses         90     107     +17     
```


| [Files](https://app.codecov.io/gh/drivendataorg/cloudpathlib/pull/403?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=drivendataorg) | Coverage Δ | |
|---|---|---|
| [cloudpathlib/azure/azblobclient.py](https://app.codecov.io/gh/drivendataorg/cloudpathlib/pull/403?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=drivendataorg#diff-Y2xvdWRwYXRobGliL2F6dXJlL2F6YmxvYmNsaWVudC5weQ==) | `94.7% <100.0%> (+0.5%)` | :arrow_up: |

... and [6 files with indirect coverage changes](https://app.codecov.io/gh/drivendataorg/cloudpathlib/pull/403/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=drivendataorg)

</details>

Patch:
diff --git a/HISTORY.md b/HISTORY.md
index 1cfa97a..52878ca 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -3,6 +3,7 @@
 ## UNRELEASED
 - Implement sliced downloads in GSClient. (Issue [#387](https://github.com/drivendataorg/cloudpathlib/issues/387), PR [#389](https://github.com/drivendataorg/cloudpathlib/pull/389))
 - Implement `as_url` with presigned parameter for all backends. (Issue [#235](https://github.com/drivendataorg/cloudpathlib/issues/235), PR [#236](https://github.com/drivendataorg/cloudpathlib/pull/236))
+- Stream to and from Azure Blob Storage. (PR [#403](https://github.com/drivendataorg/cloudpathlib/pull/403))
 
 ## 0.17.0 (2023-12-21)
 
diff --git a/cloudpathlib/azure/azblobclient.py b/cloudpathlib/azure/azblobclient.py
index dcda3e6..f161a02 100644
--- a/cloudpathlib/azure/azblobclient.py
+++ b/cloudpathlib/azure/azblobclient.py
@@ -118,6 +118,10 @@ class AzureBlobClient(Client):
 
         return properties
 
+    @staticmethod
+    def _partial_filename(local_path) -> Path:
+        return Path(str(local_path) + ".part")
+
     def _download_file(
         self, cloud_path: AzureBlobPath, local_path: Union[str, os.PathLike]
     ) -> Path:
@@ -131,7 +135,17 @@ class AzureBlobClient(Client):
 
         local_path.parent.mkdir(exist_ok=True, parents=True)
 
-        local_path.write_bytes(download_stream.content_as_bytes())
+        try:
+            partial_local_path = self._partial_filename(local_path)
+            with partial_local_path.open("wb") as data:
+                download_stream.readinto(data)
+
+            partial_local_path.replace(local_path)
+        except:  # noqa: E722
+            # remove any partial download
+            if partial_local_path.exists():
+                partial_local_path.unlink()
+            raise
 
         return local_path
 
@@ -273,7 +287,8 @@ class AzureBlobClient(Client):
 
         content_settings = ContentSettings(**extra_args)
 
-        blob.upload_blob(Path(local_path).read_bytes(), overwrite=True, content_settings=content_settings)  # type: ignore
+        with Path(local_path).open("rb") as data:
+            blob.upload_blob(data, overwrite=True, content_settings=content_settings)  # type: ignore
 
         return cloud_path
 
diff --git a/cloudpathlib/local/localclient.py b/cloudpathlib/local/localclient.py
index b3f829d..a0eadb0 100644
--- a/cloudpathlib/local/localclient.py
+++ b/cloudpathlib/local/localclient.py
@@ -5,6 +5,7 @@ import os
 from pathlib import Path, PurePosixPath
 import shutil
 from tempfile import TemporaryDirectory
+from time import sleep
 from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union
 
 from ..client import Client
@@ -63,7 +64,17 @@ class LocalClient(Client):
     def _download_file(self, cloud_path: "LocalPath", local_path: Union[str, os.PathLike]) -> Path:
         local_path = Path(local_path)
         local_path.parent.mkdir(exist_ok=True, parents=True)
-        shutil.copyfile(self._cloud_path_to_local(cloud_path), local_path)
+
+        try:
+            shutil.copyfile(self._cloud_path_to_local(cloud_path), local_path)
+        except FileNotFoundError:
+            # erroneous FileNotFoundError appears in tests sometimes; patiently insist on the parent directory existing
+            sleep(1.0)
+            local_path.parent.mkdir(exist_ok=True, parents=True)
+            sleep(1.0)
+
+            shutil.copyfile(self._cloud_path_to_local(cloud_path), local_path)
+
         return local_path
 
     def _exists(self, cloud_path: "LocalPath") -> bool:


Test patch:
diff --git a/tests/mock_clients/mock_azureblob.py b/tests/mock_clients/mock_azureblob.py
index 3ddb7ee..b07aeb0 100644
--- a/tests/mock_clients/mock_azureblob.py
+++ b/tests/mock_clients/mock_azureblob.py
@@ -111,7 +111,7 @@ class MockBlobClient:
     def upload_blob(self, data, overwrite, content_settings=None):
         path = self.root / self.key
         path.parent.mkdir(parents=True, exist_ok=True)
-        path.write_bytes(data)
+        path.write_bytes(data.read())
 
         if content_settings is not None:
             self.service_client.metadata_cache[self.root / self.key] = (
@@ -130,6 +130,9 @@ class MockStorageStreamDownloader:
     def content_as_bytes(self):
         return self.readall()
 
+    def readinto(self, buffer):
+        buffer.write(self.readall())
+
 
 class MockContainerClient:
     def __init__(self, root, container_name):
diff --git a/tests/test_azure_specific.py b/tests/test_azure_specific.py
index 7133d8f..f229ce6 100644
--- a/tests/test_azure_specific.py
+++ b/tests/test_azure_specific.py
@@ -1,5 +1,6 @@
 import os
 
+from azure.storage.blob import StorageStreamDownloader
 import pytest
 
 from urllib.parse import urlparse, parse_qs
@@ -7,6 +8,8 @@ from cloudpathlib import AzureBlobClient, AzureBlobPath
 from cloudpathlib.exceptions import MissingCredentialsError
 from cloudpathlib.local import LocalAzureBlobClient, LocalAzureBlobPath
 
+from .mock_clients.mock_azureblob import MockStorageStreamDownloader
+
 
 @pytest.mark.parametrize("path_class", [AzureBlobPath, LocalAzureBlobPath])
 def test_azureblobpath_properties(path_class, monkeypatch):
@@ -45,3 +48,34 @@ def test_as_url(azure_rig):
     assert "sp" in query_params
     assert "sr" in query_params
     assert "sig" in query_params
+
+
+def test_partial_download(azure_rig, monkeypatch):
+    p: AzureBlobPath = azure_rig.create_cloud_path("dir_0/file0_0.txt")
+
+    # no partial after successful download
+    p.read_text()  # downloads
+    assert p._local.exists()
+    assert not p.client._partial_filename(p._local).exists()
+
+    # remove cache manually
+    p._local.unlink()
+    assert not p._local.exists()
+
+    # no partial after failed download
+    with monkeypatch.context() as m:
+
+        def _patched(self, buffer):
+            buffer.write(b"partial")
+            raise Exception("boom")
+
+        if azure_rig.live_server:
+            m.setattr(StorageStreamDownloader, "readinto", _patched)
+        else:
+            m.setattr(MockStorageStreamDownloader, "readinto", _patched)
+
+        with pytest.raises(Exception):
+            p.read_text()  # downloads; should raise
+
+        assert not p._local.exists()
+        assert not p.client._partial_filename(p._local).exists()