Repository: mandarons/icloud-drive-docker (python)

Issue:
[FEATURE] Handling 503, 504 errors
**Use case**

I’ve noticed a level of instability with the container shutting down , looking in the logs I can see a few 50x errors were the cause..

  1) Service Unavailable (503)
  2) Gateway Time-out (504)

**Describe the solution you'd like**

Is it possible for iCloud-drive-docker to handle this better, rather than exit out

**Describe alternatives you've considered**

N/A

**Additional context**

Here is what I can see in the logs for the 503..
````
2023-01-19 00:43:20,013 :: INFO :: root :: sync_drive.py :: 161 :: Downloading /app/icloud/drive/Downloads/traffic_lights.png ...
2023-01-19 00:43:37,693 :: INFO :: root :: sync_drive.py :: 161 :: Downloading /app/icloud/drive/Downloads/ttyUSBSpy/core/__init__.py ...
2023-01-19 00:43:40,925 :: ERROR :: icloudpy.base :: base.py :: 186 :: Service Unavailable (503)
Traceback (most recent call last):
  File "/app/./src/main.py", line 6, in <module>
    sync.sync()
  File "/app/src/sync.py", line 65, in sync
    sync_drive.sync_drive(config=config, drive=api.drive)
  File "/app/src/sync_drive.py", line 267, in sync_drive
    return sync_directory(
  File "/app/src/sync_drive.py", line 235, in sync_directory
    sync_directory(
  File "/app/src/sync_drive.py", line 235, in sync_directory
    sync_directory(
  File "/app/src/sync_drive.py", line 235, in sync_directory
    sync_directory(
  File "/app/src/sync_drive.py", line 252, in sync_directory
    process_file(
  File "/app/src/sync_drive.py", line 183, in process_file
    if is_package(item=item):
  File "/app/src/sync_drive.py", line 153, in is_package
    with item.open(stream=True) as response:
  File "/app/venv/lib/python3.9/site-packages/icloudpy/services/drive.py", line 309, in open
    return self.connection.get_file(
  File "/app/venv/lib/python3.9/site-packages/icloudpy/services/drive.py", line 63, in get_file
    return self.session.get(data_token["url"], params=self.params, **kwargs)
  File "/app/venv/lib/python3.9/site-packages/requests/sessions.py", line 600, in get
    return self.request("GET", url, **kwargs)
  File "/app/venv/lib/python3.9/site-packages/icloudpy/base.py", line 131, in request
    self._raise_error(response.status_code, response.reason)
  File "/app/venv/lib/python3.9/site-packages/icloudpy/base.py", line 187, in _raise_error
    raise api_error
icloudpy.exceptions.ICloudPyAPIResponseException: Service Unavailable (503)
````
And this is the 504
````
2023-01-19 08:47:06,892 :: INFO :: root :: sync_drive.py :: 161 :: Downloading /app/icloud/drive/Downloads/ttyUSBSpy/core/__init__.py ...
2023-01-19 08:47:31,227 :: INFO :: root :: sync_drive.py :: 161 :: Downloading /app/icloud/drive/Downloads/veraflux-master.zip ...
2023-01-19 08:47:32,729 :: INFO :: root :: sync_drive.py :: 161 :: Downloading /app/icloud/drive/Downloads/Yeah it was great.you ok. how wo.vmg ...
2023-01-19 08:48:28,907 :: ERROR :: icloudpy.base :: base.py :: 186 :: Gateway Time-out (504)
Traceback (most recent call last):
  File "/app/./src/main.py", line 6, in <module>
    sync.sync()
  File "/app/src/sync.py", line 65, in sync
    sync_drive.sync_drive(config=config, drive=api.drive)
  File "/app/src/sync_drive.py", line 267, in sync_drive
    return sync_directory(
  File "/app/src/sync_drive.py", line 235, in sync_directory
    sync_directory(
  File "/app/src/sync_drive.py", line 252, in sync_directory
    process_file(
  File "/app/src/sync_drive.py", line 183, in process_file
    if is_package(item=item):
  File "/app/src/sync_drive.py", line 153, in is_package
    with item.open(stream=True) as response:
  File "/app/venv/lib/python3.9/site-packages/icloudpy/services/drive.py", line 309, in open
    return self.connection.get_file(
  File "/app/venv/lib/python3.9/site-packages/icloudpy/services/drive.py", line 65, in get_file
    return self.session.get(package_token["url"], params=self.params, **kwargs)
  File "/app/venv/lib/python3.9/site-packages/requests/sessions.py", line 600, in get
    return self.request("GET", url, **kwargs)
  File "/app/venv/lib/python3.9/site-packages/icloudpy/base.py", line 131, in request
    self._raise_error(response.status_code, response.reason)
  File "/app/venv/lib/python3.9/site-packages/icloudpy/base.py", line 187, in _raise_error
    raise api_error
icloudpy.exceptions.ICloudPyAPIResponseException: Gateway Time-out (504)
````

Pull request:
Handle errors raised by icloudpy and continue execution
Workaround for #113 and closes #116

Patch:
diff --git a/src/sync_drive.py b/src/sync_drive.py
index 42d21401..43ce733d 100644
--- a/src/sync_drive.py
+++ b/src/sync_drive.py
@@ -254,17 +254,21 @@ def sync_directory(
                 )
                 if not new_folder:
                     continue
-                files.add(new_folder)
-                files.update(
-                    sync_directory(
-                        drive=item,
-                        destination_path=new_folder,
-                        items=item.dir(),
-                        root=root,
-                        top=False,
-                        filters=filters,
+                try:
+                    files.add(new_folder)
+                    files.update(
+                        sync_directory(
+                            drive=item,
+                            destination_path=new_folder,
+                            items=item.dir(),
+                            root=root,
+                            top=False,
+                            filters=filters,
+                        )
                     )
-                )
+                except Exception:
+                    # Continue execution to next item, without crashing the app
+                    pass
             elif item.type == "file":
                 if wanted_parent_folder(
                     filters=filters["folders"]
@@ -273,14 +277,18 @@ def sync_directory(
                     root=root,
                     folder_path=destination_path,
                 ):
-                    process_file(
-                        item=item,
-                        destination_path=destination_path,
-                        filters=filters["file_extensions"]
-                        if filters and "file_extensions" in filters
-                        else None,
-                        files=files,
-                    )
+                    try:
+                        process_file(
+                            item=item,
+                            destination_path=destination_path,
+                            filters=filters["file_extensions"]
+                            if filters and "file_extensions" in filters
+                            else None,
+                            files=files,
+                        )
+                    except Exception:
+                        # Continue execution to next item, without crashing the app
+                        pass
         if top and remove:
             remove_obsolete(destination_path=destination_path, files=files)
     return files


Test patch:
diff --git a/tests/test_sync_drive.py b/tests/test_sync_drive.py
index 94606b77..3592fd37 100644
--- a/tests/test_sync_drive.py
+++ b/tests/test_sync_drive.py
@@ -6,6 +6,8 @@ import shutil
 import unittest
 from unittest.mock import patch
 
+from icloudpy.exceptions import ICloudPyAPIResponseException
+
 import tests
 from src import LOGGER, read_config, sync_drive
 from tests import DATA_DIR, data
@@ -26,6 +28,7 @@ class TestSyncDrive(unittest.TestCase):
         )
         self.drive = self.service.drive
         self.items = self.drive.dir()
+        self.folder_item = self.drive[self.items[5]]
         self.file_item = self.drive[self.items[4]]["Test"]["Scanned document 1.pdf"]
         self.package_item = self.drive[self.items[6]]["Sample"]["Project.band"]
         self.package_item_nested = self.drive[self.items[6]]["Sample"]["ms.band"]
@@ -1029,3 +1032,50 @@ class TestSyncDrive(unittest.TestCase):
         self.assertFalse(
             sync_drive.process_package(local_file=os.path.join(DATA_DIR, "medium.jpeg"))
         )
+
+    def test_execution_continuation_on_icloudpy_exception(self):
+        """Test for icloudpy exception."""
+        with patch.object(self.file_item, "open") as mocked_file_method, patch.object(
+            self.folder_item, "dir"
+        ) as mocked_folder_method:
+            mocked_file_method.side_effect = (
+                mocked_folder_method.side_effect
+            ) = ICloudPyAPIResponseException("Exception occurred.")
+            filters = dict(self.filters)
+            filters["folders"].append("unwanted")
+            actual = sync_drive.sync_directory(
+                drive=self.drive,
+                destination_path=self.destination_path,
+                root=self.root,
+                items=self.drive.dir(),
+                top=True,
+                filters=filters,
+                remove=False,
+            )
+            self.assertTrue(len(actual) == 23)
+            self.assertTrue(
+                os.path.isdir(os.path.join(self.destination_path, "icloudpy"))
+            )
+            self.assertTrue(
+                os.path.isdir(os.path.join(self.destination_path, "icloudpy", "Test"))
+            )
+            self.assertTrue(
+                os.path.isfile(
+                    os.path.join(
+                        self.destination_path,
+                        "icloudpy",
+                        "Test",
+                        "Document scanne 2.pdf",
+                    )
+                )
+            )
+            self.assertFalse(
+                os.path.isfile(
+                    os.path.join(
+                        self.destination_path,
+                        "icloudpy",
+                        "Test",
+                        "Scanned document 1.pdf",
+                    )
+                )
+            )