Repository: level12/keg-storage

Issue:
Implement method(s) to natively copy objects in S3 and local storage backends
Need a way to natively copy objects so that files don't need to be downloaded and re-uploaded in order to duplicate.

Discussion:

Patch:
diff --git a/keg_storage/backends/base.py b/keg_storage/backends/base.py
index dedaa6b..dcc51ba 100644
--- a/keg_storage/backends/base.py
+++ b/keg_storage/backends/base.py
@@ -158,6 +158,12 @@ class StorageBackend:
         """
         raise NotImplementedError()
 
+    def copy(self, path: str, new_path: str):
+        """
+        Copy the remote file specified by `path` to `new_path`.
+        """
+        raise NotImplementedError()
+
     def link_to(
             self,
             path: str,
diff --git a/keg_storage/backends/filesystem.py b/keg_storage/backends/filesystem.py
index a0a7d04..49c7171 100644
--- a/keg_storage/backends/filesystem.py
+++ b/keg_storage/backends/filesystem.py
@@ -122,6 +122,9 @@ class LocalFSStorage(base.InternalLinksStorageBackend):
 
         return LocalFSFile(path, mode)
 
+    def copy(self, path: str, new_path: str):
+        return self.put(path, new_path)
+
     def delete(self, path: str):
         self._validate_path(path)
         path = self._resolve_path(path)
diff --git a/keg_storage/backends/s3.py b/keg_storage/backends/s3.py
index 96952a2..ca6b2b2 100644
--- a/keg_storage/backends/s3.py
+++ b/keg_storage/backends/s3.py
@@ -248,6 +248,16 @@ class S3Storage(StorageBackend):
         else:
             raise ValueError('Unsupported mode. Accepted modes are FileMode.read or FileMode.write')
 
+    def copy(self, current_file, new_file):
+        self.client.copy_object(
+            CopySource={
+                'Bucket': self.bucket,
+                'Key': current_file
+            },
+            Bucket=self.bucket,
+            Key=new_file
+        )
+
     def delete(self, path):
         self.client.delete_object(
             Bucket=self.bucket,
diff --git a/keg_storage/cli.py b/keg_storage/cli.py
index c1abd3f..4aa39bc 100644
--- a/keg_storage/cli.py
+++ b/keg_storage/cli.py
@@ -84,6 +84,15 @@ def storage_put(ctx, file, key):
                err=True)
 
 
+@storage.command('copy')
+@click.argument('src_key')
+@click.argument('dest_key')
+@click.pass_context
+def storage_copy(ctx, src_key, dest_key):
+    ctx.obj.data['storage'].copy(src_key, dest_key)
+    click.echo(f"Copied {src_key} to {dest_key}.")
+
+
 @storage.command('delete')
 @click.argument('path')
 @click.pass_context


Test patch:
diff --git a/keg_storage/tests/test_backend_filesystem.py b/keg_storage/tests/test_backend_filesystem.py
index 168f922..1f82295 100644
--- a/keg_storage/tests/test_backend_filesystem.py
+++ b/keg_storage/tests/test_backend_filesystem.py
@@ -193,6 +193,22 @@ class TestLocalFSStorage:
             assert fp.read() == file_data2
         assert f.fp.closed is True
 
+    def test_copy_file(self, tmp_path: pathlib.Path):
+        root = tmp_path.joinpath('root')
+        root.mkdir()
+
+        file_path = root.joinpath('file.txt')
+        file_path_copy = root.joinpath('file2.txt')
+        fs = backends.LocalFSStorage(root)
+        file_data1 = os.urandom(100)
+
+        with fs.open('file.txt', backends.FileMode.write) as f:
+            f.write(file_data1)
+        fs.copy(str(file_path), str(file_path_copy))
+
+        with file_path_copy.open('rb') as fp:
+            assert fp.read() == file_data1
+
     def test_open_for_writing_creates_directories(self, tmp_path: pathlib.Path):
         root = tmp_path.joinpath('root')
         root.mkdir()
diff --git a/keg_storage/tests/test_backend_s3.py b/keg_storage/tests/test_backend_s3.py
index 220add6..971f1d2 100644
--- a/keg_storage/tests/test_backend_s3.py
+++ b/keg_storage/tests/test_backend_s3.py
@@ -285,6 +285,19 @@ class TestS3Storage:
             }
         )
 
+    def test_copy(self, m_boto):
+        s3 = backends.S3Storage('bucket', aws_region='us-east-1')
+        s3.copy('foo/bar', 'foo/baz')
+
+        s3.client.copy_object.assert_called_once_with(
+            CopySource={
+                'Bucket': 'bucket',
+                'Key': 'foo/bar'
+            },
+            Bucket='bucket',
+            Key='foo/baz'
+        )
+
     def test_write_abort(self, m_boto):
         m_client = mock.MagicMock()
         m_client.create_multipart_upload.return_value = {'UploadId': 'upload-id'}