Repository: kedro-org/kedro

Issue:
Implement `delete_existing_files_on_save` method for `ParititionedDataSet` and `MatplotlibWriter` 
## Description
Several users have discussed the fact that `PartitionedDataSet` and `MatplotlibWriter` datasets do not clear the contents of the directories they write to when saving. A discussion can be found on this issue https://github.com/quantumblacklabs/kedro/issues/868 

We should keep the current behaviour as default, but provide an extra argument that the user can explicitly ask Kedro to wipe the contents of a directory at save time.

## Possible Implementation
(Optional) Suggest an idea for implementing the addition or change.

`fsspec` library provides a `delete()` method which can be used for this purposes.

## Possible Alternatives
(Optional) Describe any alternative solutions or features you've considered.

We could simply update the documentation to highlight that current default behaviour does not delete old data.


Discussion:
I can take a stab at this. Can we call it `overwrite` instead of `delete_existing_files_on_save`? 😂

Resolving patch:
diff --git a/kedro/extras/datasets/matplotlib/matplotlib_writer.py b/kedro/extras/datasets/matplotlib/matplotlib_writer.py
--- a/kedro/extras/datasets/matplotlib/matplotlib_writer.py
+++ b/kedro/extras/datasets/matplotlib/matplotlib_writer.py
@@ -1,5 +1,5 @@
-"""``MatplotlibWriter`` saves matplotlib objects as image file(s) to an underlying
-filesystem (e.g. local, S3, GCS)."""
+"""``MatplotlibWriter`` saves one or more Matplotlib objects as image
+files to an underlying filesystem (e.g. local, S3, GCS)."""
 
 import io
 from copy import deepcopy
@@ -19,8 +19,8 @@
 
 
 class MatplotlibWriter(AbstractVersionedDataSet):
-    """``MatplotlibWriter`` saves matplotlib objects to image file(s) in an underlying
-    filesystem (e.g. local, S3, GCS).
+    """``MatplotlibWriter`` saves one or more Matplotlib objects as
+    image files to an underlying filesystem (e.g. local, S3, GCS).
 
     Example:
     ::
@@ -70,13 +70,14 @@ def __init__(
         credentials: Dict[str, Any] = None,
         save_args: Dict[str, Any] = None,
         version: Version = None,
+        overwrite: bool = False,
     ) -> None:
         """Creates a new instance of ``MatplotlibWriter``.
 
         Args:
-            filepath: Filepath in POSIX format to a matplot object file(s) prefixed with a protocol
-                like `s3://`. If prefix is not provided, `file` protocol (local filesystem) will be
-                used. The prefix should be any protocol supported by ``fsspec``.
+            filepath: Filepath in POSIX format to save Matplotlib objects to, prefixed with a
+                protocol like `s3://`. If prefix is not provided, `file` protocol (local filesystem)
+                will be used. The prefix should be any protocol supported by ``fsspec``.
             fs_args: Extra arguments to pass into underlying filesystem class constructor
                 (e.g. `{"project": "my-project"}` for ``GCSFileSystem``), as well as
                 to pass to the filesystem's `open` method through nested key `open_args_save`.
@@ -92,6 +93,9 @@ def __init__(
                 ``kedro.io.core.Version``. If its ``load`` attribute is
                 None, the latest version will be loaded. If its ``save``
                 attribute is None, save version will be autogenerated.
+            overwrite: If True, any existing image files will be removed.
+                Only relevant when saving multiple Matplotlib objects at
+                once.
         """
         _credentials = deepcopy(credentials) or {}
         _fs_args = deepcopy(fs_args) or {}
@@ -119,6 +123,8 @@ def __init__(
         if save_args is not None:
             self._save_args.update(save_args)
 
+        self._overwrite = overwrite
+
     def _describe(self) -> Dict[str, Any]:
         return dict(
             filepath=self._filepath,
@@ -135,6 +141,13 @@ def _save(
     ) -> None:
         save_path = self._get_save_path()
 
+        if (
+            isinstance(data, (list, dict))
+            and self._overwrite
+            and self._fs.exists(self._filepath)
+        ):
+            self._fs.rm(self._filepath, recursive=True)
+
         if isinstance(data, list):
             for index, plot in enumerate(data):
                 full_key_path = get_filepath_str(
diff --git a/kedro/io/partitioned_data_set.py b/kedro/io/partitioned_data_set.py
--- a/kedro/io/partitioned_data_set.py
+++ b/kedro/io/partitioned_data_set.py
@@ -103,6 +103,7 @@ def __init__(  # pylint: disable=too-many-arguments
         credentials: Dict[str, Any] = None,
         load_args: Dict[str, Any] = None,
         fs_args: Dict[str, Any] = None,
+        overwrite: bool = False,
     ):
         """Creates a new instance of ``PartitionedDataSet``.
 
@@ -140,6 +141,7 @@ def __init__(  # pylint: disable=too-many-arguments
                 the filesystem implementation.
             fs_args: Extra arguments to pass into underlying filesystem class constructor
                 (e.g. `{"project": "my-project"}` for ``GCSFileSystem``)
+            overwrite: If True, any existing partitions will be removed.
 
         Raises:
             DataSetError: If versioning is enabled for the underlying dataset.
@@ -151,6 +153,7 @@ def __init__(  # pylint: disable=too-many-arguments
 
         self._path = path
         self._filename_suffix = filename_suffix
+        self._overwrite = overwrite
         self._protocol = infer_storage_options(self._path)["protocol"]
         self._partition_cache = Cache(maxsize=1)
 
@@ -255,6 +258,9 @@ def _load(self) -> Dict[str, Callable[[], Any]]:
         return partitions
 
     def _save(self, data: Dict[str, Any]) -> None:
+        if self._overwrite and self._filesystem.exists(self._path):
+            self._filesystem.rm(self._path, recursive=True)
+
         for partition_id, partition_data in sorted(data.items()):
             kwargs = deepcopy(self._dataset_config)
             partition = self._partition_to_path(partition_id)


Test patch:
diff --git a/tests/extras/datasets/matplotlib/test_matplotlib_writer.py b/tests/extras/datasets/matplotlib/test_matplotlib_writer.py
--- a/tests/extras/datasets/matplotlib/test_matplotlib_writer.py
+++ b/tests/extras/datasets/matplotlib/test_matplotlib_writer.py
@@ -97,15 +97,21 @@ def s3fs_cleanup():
     S3FileSystem.cachable = False
 
 
+@pytest.fixture(params=[False])
+def overwrite(request):
+    return request.param
+
+
 @pytest.fixture
 def plot_writer(
-    mocked_s3_bucket, fs_args, save_args
+    mocked_s3_bucket, fs_args, save_args, overwrite
 ):  # pylint: disable=unused-argument
     return MatplotlibWriter(
         filepath=FULL_PATH,
         credentials=AWS_CREDENTIALS,
         fs_args=fs_args,
         save_args=save_args,
+        overwrite=overwrite,
     )
 
 
@@ -176,8 +182,30 @@ def test_dict_save(self, tmp_path, mock_dict_plot, plot_writer, mocked_s3_bucket
 
             assert actual_filepath.read_bytes() == download_path.read_bytes()
 
+    @pytest.mark.parametrize(
+        "overwrite,expected_num_plots", [(False, 8), (True, 3)], indirect=["overwrite"]
+    )
+    def test_overwrite(
+        self,
+        mock_list_plot,
+        mock_dict_plot,
+        plot_writer,
+        mocked_s3_bucket,
+        expected_num_plots,
+    ):
+        """Test saving dictionary of plots after list of plots to S3."""
+
+        plot_writer.save(mock_list_plot)
+        plot_writer.save(mock_dict_plot)
+
+        response = mocked_s3_bucket.list_objects(Bucket=BUCKET_NAME)
+        saved_plots = {obj["Key"] for obj in response["Contents"]}
+
+        assert {f"{KEY_PATH}/{colour}" for colour in COLOUR_LIST} <= saved_plots
+        assert len(saved_plots) == expected_num_plots
+
     def test_fs_args(self, tmp_path, mock_single_plot, mocked_encrypted_s3_bucket):
-        """Test writing to encrypted bucket"""
+        """Test writing to encrypted bucket."""
         normal_encryped_writer = MatplotlibWriter(
             fs_args={"s3_additional_kwargs": {"ServerSideEncryption": "AES256"}},
             filepath=FULL_PATH,
diff --git a/tests/io/test_partitioned_dataset.py b/tests/io/test_partitioned_dataset.py
--- a/tests/io/test_partitioned_dataset.py
+++ b/tests/io/test_partitioned_dataset.py
@@ -121,6 +121,19 @@ def test_save_invalidates_cache(self, local_csvs, mocker):
         assert new_partition not in first_load
         assert new_partition in second_load
 
+    @pytest.mark.parametrize("overwrite,expected_num_parts", [(False, 6), (True, 1)])
+    def test_overwrite(self, local_csvs, overwrite, expected_num_parts):
+        pds = PartitionedDataSet(
+            str(local_csvs), "pandas.CSVDataSet", overwrite=overwrite
+        )
+        original_data = pd.DataFrame({"foo": 42, "bar": ["a", "b", None]})
+        part_id = "new/data"
+        pds.save({part_id: original_data})
+        loaded_partitions = pds.load()
+
+        assert part_id in loaded_partitions
+        assert len(loaded_partitions.keys()) == expected_num_parts
+
     def test_release_instance_cache(self, local_csvs):
         """Test that cache invalidation does not affect other instances"""
         ds_a = PartitionedDataSet(str(local_csvs), "pandas.CSVDataSet")