Repository: mandarons/icloud-drive-docker
Pull request #129 (2023-03-03 13:14:36)

Make use of remove_obsolete config option for photos
Currently remove_obsolete_files for photos is unused.

Example config (https://github.com/mandarons/icloud-drive-docker/blob/main/config.yaml#L46) shows it.


Patch:
diff --git a/src/sync_photos.py b/src/sync_photos.py
index f3740d74..bc27f8eb 100644
--- a/src/sync_photos.py
+++ b/src/sync_photos.py
@@ -4,6 +4,7 @@ import base64
 import os
 import shutil
 import time
+from pathlib import Path
 
 from icloudpy import exceptions
 
@@ -71,7 +72,7 @@ def download_photo(photo, file_size, destination_path):
     return True
 
 
-def process_photo(photo, file_size, destination_path):
+def process_photo(photo, file_size, destination_path, files):
     """Process photo details."""
     photo_path = generate_file_name(
         photo=photo, file_size=file_size, destination_path=destination_path
@@ -81,21 +82,24 @@ def process_photo(photo, file_size, destination_path):
             f"File size {file_size} not found on server. Skipping the photo {photo_path} ..."
         )
         return False
+    if not files is None:
+        files.add(photo_path)
     if photo_exists(photo, file_size, photo_path):
         return False
     download_photo(photo, file_size, photo_path)
     return True
 
 
-def sync_album(album, destination_path, file_sizes, extensions=None):
+def sync_album(album, destination_path, file_sizes, extensions=None, files=None):
     """Sync given album."""
     if album is None or destination_path is None or file_sizes is None:
         return None
     os.makedirs(destination_path, exist_ok=True)
+    LOGGER.info(f"Syncing {album.title}")
     for photo in album:
         if photo_wanted(photo, extensions):
             for file_size in file_sizes:
-                process_photo(photo, file_size, destination_path)
+                process_photo(photo, file_size, destination_path, files)
         else:
             LOGGER.debug(f"Skipping the unwanted photo {photo.filename}.")
     for subalbum in album.subalbums:
@@ -104,14 +108,31 @@ def sync_album(album, destination_path, file_sizes, extensions=None):
             os.path.join(destination_path, subalbum),
             file_sizes,
             extensions,
+            files,
         )
     return True
 
 
+def remove_obsolete(destination_path, files):
+    """Remove local obsolete file."""
+    removed_paths = set()
+    if not (destination_path and files is not None):
+        return removed_paths
+    for path in Path(destination_path).rglob("*"):
+        local_file = str(path.absolute())
+        if local_file not in files:
+            if path.is_file():
+                LOGGER.info(f"Removing {local_file} ...")
+                path.unlink(missing_ok=True)
+                removed_paths.add(local_file)
+    return removed_paths
+
+
 def sync_photos(config, photos):
     """Sync all photos."""
     destination_path = config_parser.prepare_photos_destination(config=config)
     filters = config_parser.get_photos_filters(config=config)
+    files = set()
     if filters["albums"]:
         for album in iter(filters["albums"]):
             sync_album(
@@ -119,6 +140,7 @@ def sync_photos(config, photos):
                 destination_path=os.path.join(destination_path, album),
                 file_sizes=filters["file_sizes"],
                 extensions=filters["extensions"],
+                files=files,
             )
     else:
         sync_album(
@@ -126,8 +148,12 @@ def sync_photos(config, photos):
             destination_path=os.path.join(destination_path, "all"),
             file_sizes=filters["file_sizes"],
             extensions=filters["extensions"],
+            files=files,
         )
 
+    if config_parser.get_photos_remove_obsolete(config=config):
+        remove_obsolete(destination_path, files)
+
 
 # def enable_debug():
 #     import contextlib


Test patch:
diff --git a/tests/test_sync_photos.py b/tests/test_sync_photos.py
index 85d9e8af..15d74d78 100644
--- a/tests/test_sync_photos.py
+++ b/tests/test_sync_photos.py
@@ -244,6 +244,8 @@ class TestSyncPhotos(unittest.TestCase):
                 next((s for s in captured[1] if "Downloading /" in s), None)
             )
 
+        self.assertFalse(os.path.exists(os.path.join(album_1_path, "IMG_3148.JPG")))
+
         # Rename previous __original files - upgrade to newer version
         os.rename(
             os.path.join(
@@ -262,6 +264,94 @@ class TestSyncPhotos(unittest.TestCase):
                 next((s for s in captured[1] if "Downloading /" in s), None)
             )
 
+    @patch(target="keyring.get_password", return_value=data.VALID_PASSWORD)
+    @patch(
+        target="src.config_parser.get_username", return_value=data.AUTHENTICATED_USER
+    )
+    @patch("icloudpy.ICloudPyService")
+    @patch("src.read_config")
+    def test_sync_photos_rename_original_photos_obsolete_false(
+        self,
+        mock_read_config,
+        mock_service,
+        mock_get_username,
+        mock_get_password,
+    ):
+        """Test for renaming of previously downloaded original photos."""
+        mock_service = self.service
+        config = self.config.copy()
+        config["photos"]["destination"] = self.destination_path
+        config["photos"]["remove_obsolete"] = False
+        mock_read_config.return_value = config
+        album_1_path = os.path.join(
+            self.destination_path, config["photos"]["filters"]["albums"][1]
+        )
+        sync_photos.sync_photos(config=config, photos=mock_service.photos)
+
+        # Rename previous original files - upgrade to newer version
+        os.rename(
+            os.path.join(
+                album_1_path,
+                "IMG_3148__original__QVZ4My9WS2tiV1BkTmJXdzY4bXJXelN1ZW1YZw==.JPG",
+            ),
+            os.path.join(album_1_path, "delete_me.JPG"),
+        )
+
+        sync_photos.sync_photos(config=config, photos=mock_service.photos)
+
+        self.assertTrue(os.path.exists(os.path.join(album_1_path, "delete_me.JPG")))
+
+    @patch(target="keyring.get_password", return_value=data.VALID_PASSWORD)
+    @patch(
+        target="src.config_parser.get_username", return_value=data.AUTHENTICATED_USER
+    )
+    @patch("icloudpy.ICloudPyService")
+    @patch("src.read_config")
+    def test_sync_photos_rename_original_photos_obsolete_true(
+        self,
+        mock_read_config,
+        mock_service,
+        mock_get_username,
+        mock_get_password,
+    ):
+        """Test for renaming of previously downloaded original photos."""
+        mock_service = self.service
+        config = self.config.copy()
+        config["photos"]["destination"] = self.destination_path
+        config["photos"]["remove_obsolete"] = True
+        mock_read_config.return_value = config
+        album_1_path = os.path.join(
+            self.destination_path, config["photos"]["filters"]["albums"][1]
+        )
+        sync_photos.sync_photos(config=config, photos=mock_service.photos)
+
+        # Rename previous original files - upgrade to newer version
+        os.rename(
+            os.path.join(
+                album_1_path,
+                "IMG_3148__original__QVZ4My9WS2tiV1BkTmJXdzY4bXJXelN1ZW1YZw==.JPG",
+            ),
+            os.path.join(album_1_path, "delete_me.JPG"),
+        )
+
+        sync_photos.sync_photos(config=config, photos=mock_service.photos)
+
+        self.assertFalse(os.path.exists(os.path.join(album_1_path, "delete_me.JPG")))
+
+    def test_remove_obsolete_none_destination_path(self):
+        """Test for destination path as None."""
+        self.assertTrue(
+            len(sync_photos.remove_obsolete(destination_path=None, files=set())) == 0
+        )
+
+    def test_remove_obsolete_none_files(self):
+        """Test for files as None."""
+        obsolete_path = os.path.join(self.destination_path, "obsolete")
+        self.assertTrue(
+            len(sync_photos.remove_obsolete(destination_path=obsolete_path, files=None))
+            == 0
+        )
+
     @patch(target="keyring.get_password", return_value=data.VALID_PASSWORD)
     @patch(
         target="src.config_parser.get_username", return_value=data.AUTHENTICATED_USER
@@ -372,6 +462,7 @@ class TestSyncPhotos(unittest.TestCase):
                 photo=MockPhoto(),
                 file_size="medium",
                 destination_path=self.destination_path,
+                files=None,
             )
         )
 
@@ -396,6 +487,7 @@ class TestSyncPhotos(unittest.TestCase):
                 photo=MockPhoto(),
                 file_size="thumb",
                 destination_path=self.destination_path,
+                files=None,
             )
         )