Repository: PyFilesystem/pyfilesystem2
Pull request #22 (2017-02-10 15:02:21)

Add copy_if_newer parameter to functions in copy.py
Added copy_if_newer parameter to all functions in copy.py. 
Files are copied only if destination files don't exist or sources are newer then destination files. 
Directories are always copied. 
In case time of modification for files cannot be determined files are safely copied.

Moreover I added geturl method to osfs returning "file://" + self.getsyspath(path).



Patch:
diff --git a/.gitignore b/.gitignore
index 9eafa33..3df8aa5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -89,3 +89,8 @@ ENV/
 
 # Rope project settings
 .ropeproject
+
+
+#PyCharm
+
+.idea/
diff --git a/fs/copy.py b/fs/copy.py
index 155810a..4a52a84 100644
--- a/fs/copy.py
+++ b/fs/copy.py
@@ -13,9 +13,10 @@ from .path import abspath
 from .path import combine
 from .path import frombase
 from .path import normpath
+from .errors import FSError
 
 
-def copy_fs(src_fs, dst_fs, walker=None):
+def copy_fs(src_fs, dst_fs, walker=None, on_copy=None):
     """
     Copy the contents of one filesystem to another.
 
@@ -29,17 +30,72 @@ def copy_fs(src_fs, dst_fs, walker=None):
         in ``src_fs``. Set this if you only want to consider a sub-set
         of the resources in ``src_fs``.
     :type walker: :class:`~fs.walk.Walker`
+    :param on_copy: A function callback called after a single file copy is executed.
+    :type on_copy: Function, with signature ``(src_fs, src_path, dst_fs, dst_path)``.
+    """
+    return copy_dir(src_fs, '/', dst_fs, '/', walker=walker, on_copy=on_copy)
+
+
+def copy_fs_if_newer(src_fs, dst_fs, walker=None, on_copy=None):
+    """
+    Copy the contents of one filesystem to another. If both source and destination files exist,
+    the copy is executed only if the source file is newer than the destination file.
+    In case modification times of source or destination files are not available,
+    copy file is always executed.
+
+    :param src_fs: Source filesystem.
+    :type src_fs: :type src_fs: FS URL or instance
+    :param src_path: A path to a directory on ``src_fs``.
+    :type src_path: str
+    :param dst_fs: Destination filesystem.
+    :type dst_fs: FS URL or instance
+    :param walker: A walker object that will be used to scan for files
+        in ``src_fs``. Set this if you only want to consider a sub-set
+        of the resources in ``src_fs``.
+    :type walker: :class:`~fs.walk.Walker`
+    :param on_copy: A function callback called after a single file copy is executed.
+    :type on_copy: Function, with signature ``(src_fs, src_path, dst_fs, dst_path)``.
+    """
+    return copy_dir_if_newer(src_fs, '/', dst_fs, '/', walker=walker, on_copy=on_copy)
+
 
+def _source_is_newer(src_fs, src_path, dst_fs, dst_path):
     """
-    copy_dir(src_fs, '/', dst_fs, '/', walker=walker)
+       Determine if source file is newer than destination file.
+
+       :param src_fs: Source filesystem.
+       :type src_fs: :type src_fs: FS URL or instance
+       :param src_path: A path to a directory on ``src_fs``.
+       :type src_path: str
+       :param dst_fs: Destination filesystem.
+       :type dst_fs: FS URL or instance
+       :returns: True if source file is newer than destination file or
+            file modification time cannot be determined. False otherwise.
+       """
+    try:
+        if not dst_fs.exists(dst_path):
+            return True
+        else:
+            namespace = ('details', 'modified')
+
+            src_modified = src_fs.getinfo(src_path, namespace).modified
+            if src_modified is None:
+                return True
+
+            dst_modified = dst_fs.getinfo(dst_path, namespace).modified
+            if dst_modified is None:
+                return True
 
+            return src_modified > dst_modified
+    except FSError:
+        #todo: should log something here
+        #log.error("cannot determine if source file " + src_path + " is newer than destination file " + dst_path + ", thus safely copy the file')
+        return True
 
 def copy_file(src_fs, src_path, dst_fs, dst_path):
     """
     Copy a file from one filesystem to another.
-
-    If the destination exists, and is a file, it will be first
-    truncated.
+    If the destination exists, and is a file, it will be first truncated.
 
     :param src_fs: Source filesystem.
     :type src_fs: FS URL or instance
@@ -49,6 +105,7 @@ def copy_file(src_fs, src_path, dst_fs, dst_path):
     :type dst_fs: FS URL or instance
     :param dst_path: Path to a file on ``dst_fs``.
     :type dst_path: str
+    :returns: True if the file copy was executed, False otherwise.
 
     """
     with manage_fs(src_fs, writeable=False) as src_fs:
@@ -56,12 +113,57 @@ def copy_file(src_fs, src_path, dst_fs, dst_path):
             if src_fs is dst_fs:
                 # Same filesystem, so we can do a potentially optimized copy
                 src_fs.copy(src_path, dst_path, overwrite=True)
+                return True
             else:
                 # Standard copy
                 with src_fs.lock(), dst_fs.lock():
                     with src_fs.open(src_path, 'rb') as read_file:
                         # There may be an optimized copy available on dst_fs
                         dst_fs.setbinfile(dst_path, read_file)
+                        return True
+
+
+
+def copy_file_if_newer(src_fs, src_path, dst_fs, dst_path):
+    """
+    Copy a file from one filesystem to another.
+    If the destination exists, and is a file, it will be first truncated.
+    If both source and destination files exist,
+    the copy is executed only if the source file is newer than the destination file.
+    In case modification times of source or destination files are not available,
+    copy is always executed.
+
+    :param src_fs: Source filesystem.
+    :type src_fs: FS URL or instance
+    :param src_path: Path to a file on ``src_fs``.
+    :type src_path: str
+    :param dst_fs: Destination filesystem.
+    :type dst_fs: FS URL or instance
+    :param dst_path: Path to a file on ``dst_fs``.
+    :type dst_path: str
+    :returns: True if the file copy was executed, False otherwise.
+
+    """
+    with manage_fs(src_fs, writeable=False) as src_fs:
+        with manage_fs(dst_fs, create=True) as dst_fs:
+            if src_fs is dst_fs:
+                # Same filesystem, so we can do a potentially optimized copy
+                if _source_is_newer(src_fs, src_path, dst_fs, dst_path):
+                    src_fs.copy(src_path, dst_path, overwrite=True)
+                    return True
+                else:
+                    return False
+            else:
+                # Standard copy
+                with src_fs.lock(), dst_fs.lock():
+                    if _source_is_newer(src_fs, src_path, dst_fs, dst_path):
+                        with src_fs.open(src_path, 'rb') as read_file:
+                            # There may be an optimized copy available on dst_fs
+                            dst_fs.setbinfile(dst_path, read_file)
+                            return True
+                    else:
+                        return False
+
 
 
 def copy_structure(src_fs, dst_fs, walker=None):
@@ -86,7 +188,7 @@ def copy_structure(src_fs, dst_fs, walker=None):
                     dst_fs.makedir(dir_path, recreate=True)
 
 
-def copy_dir(src_fs, src_path, dst_fs, dst_path, walker=None):
+def copy_dir(src_fs, src_path, dst_fs, dst_path, walker=None, on_copy=None):
     """
     Copy a directory from one filesystem to another.
 
@@ -101,13 +203,14 @@ def copy_dir(src_fs, src_path, dst_fs, dst_path, walker=None):
         in ``src_fs``. Set this if you only want to consider a sub-set
         of the resources in ``src_fs``.
     :type walker: :class:`~fs.walk.Walker`
+    :param on_copy: A function callback called after a single file copy is executed.
+    :type on_copy: Function, with signature ``(src_fs, src_path, dst_fs, dst_path)``.
 
     """
 
     walker = walker or Walker()
     _src_path = abspath(normpath(src_path))
     _dst_path = abspath(normpath(dst_path))
-
     with manage_fs(src_fs, writeable=False) as src_fs:
         with manage_fs(dst_fs, create=True) as dst_fs:
             with src_fs.lock(), dst_fs.lock():
@@ -123,9 +226,90 @@ def copy_dir(src_fs, src_path, dst_fs, dst_path, walker=None):
                             recreate=True
                         )
                     for info in files:
-                        copy_file(
+                        file_copied = copy_file(
                             src_fs,
                             info.make_path(dir_path),
                             dst_fs,
                             info.make_path(copy_path)
                         )
+                        if file_copied:
+                            if on_copy is not None:
+                                on_copy(src_fs, dir_path, dst_fs, copy_path)
+
+
+def copy_dir_if_newer(src_fs, src_path, dst_fs, dst_path, walker=None, on_copy=None):
+    """
+    Copy a directory from one filesystem to another. If both source and destination files exist,
+    the copy is executed only if the source file is newer than the destination file.
+    In case modification times of source or destination files are not available,
+    copy is always executed.
+
+    :param src_fs: Source filesystem.
+    :type src_fs: FS URL or instance
+    :param src_path: A path to a directory on ``src_fs``.
+    :type src_path: str
+    :param dst_fs: Destination filesystem.
+    :type dst_fs: FS URL or instance
+    :param str dst_path: A path to a directory on ``dst_fs``.
+    :param walker: A walker object that will be used to scan for files
+        in ``src_fs``. Set this if you only want to consider a sub-set
+        of the resources in ``src_fs``.
+    :type walker: :class:`~fs.walk.Walker`
+    :param on_copy: A function callback called after a single file copy is executed.
+    :type on_copy: Function, with signature ``(src_fs, src_path, dst_fs, dst_path)``.
+
+    """
+    walker = walker or Walker()
+    _src_path = abspath(normpath(src_path))
+    _dst_path = abspath(normpath(dst_path))
+    with manage_fs(src_fs, writeable=False) as src_fs:
+        with manage_fs(dst_fs, create=True) as dst_fs:
+            with src_fs.lock(), dst_fs.lock():
+                dst_fs.makedir(_dst_path, recreate=True)
+                namespace = ('details', 'modified')
+                dst_state = {
+                    filepath: info
+                    for filepath, info in walker.info(dst_fs, _dst_path, namespace)
+                    if info.is_file
+                }
+                src_state = [
+                    (filepath, info) for filepath, info in walker.info(src_fs, _src_path, namespace)
+                ]
+                for dir_path, copy_info in src_state:
+                    copy_path = combine(
+                        _dst_path,
+                        frombase(_src_path, dir_path)
+                    )
+                    if copy_info.is_dir:
+                        dst_fs.makedir(copy_path, recreate=True)
+                    if copy_info.is_file:
+                        do_copy = False
+                        if dir_path not in dst_state:
+                            #dst file is not present
+                            do_copy = True
+                        else:
+                            #dst file is present, try to figure out if copy is necessary
+                            src_modified = copy_info.modified
+                            if src_modified is None:
+                                #cannot retrieve src modified timestamp => file will be copied
+                                do_copy = True
+                            dst_modified = dst_state[dir_path].modified
+                            if dst_modified is None:
+                                #cannot retrieve dst modified timestamp => file will be copied
+                                do_copy = True
+                            if src_modified > dst_modified:
+                                #src file is newer than dst file => file will be copied
+                                do_copy = True
+                        if do_copy:
+                            file_copied = copy_file(
+                                src_fs,
+                                dir_path,
+                                dst_fs,
+                                copy_path
+                            )
+                            if file_copied:
+                                if on_copy is not None:
+                                    on_copy(src_fs, dir_path, dst_fs, copy_path)
+
+
+
diff --git a/fs/ftpfs.py b/fs/ftpfs.py
index 5d6203d..2c75ff7 100644
--- a/fs/ftpfs.py
+++ b/fs/ftpfs.py
@@ -272,7 +272,7 @@ class FTPFS(FS):
 
     def _open_ftp(self):
         _ftp = FTP()
-        #_ftp.set_debuglevel(2)
+        _ftp.set_debuglevel(0)
         with ftp_errors(self):
             _ftp.connect(self.host, self.port, self.timeout)
             _ftp.login(self.user, self.passwd, self.acct)
diff --git a/fs/osfs.py b/fs/osfs.py
index 6829348..d200981 100644
--- a/fs/osfs.py
+++ b/fs/osfs.py
@@ -27,7 +27,7 @@ from .path import abspath, basename, normpath
 from .permissions import Permissions
 from .error_tools import convert_os_errors
 from .mode import Mode, validate_open_mode
-
+from .errors import NoURL
 
 log = logging.getLogger('fs.osfs')
 
@@ -302,6 +302,11 @@ class OSFS(FS):
         sys_path = self._to_sys_path(path)
         return sys_path
 
+    def geturl(self, path, purpose='download'):
+        if purpose != 'download':
+            raise NoURL(path, purpose)
+        return "file://" + self.getsyspath(path)
+
     def gettype(self, path):
         self.check()
         sys_path = self._to_sys_path(path)


Test patch:
diff --git a/tests/test_copy.py b/tests/test_copy.py
index 6cb045c..794e4b6 100644
--- a/tests/test_copy.py
+++ b/tests/test_copy.py
@@ -1,6 +1,12 @@
 from __future__ import unicode_literals
 
+import errno
+import os
+import time
 import unittest
+import tempfile
+import shutil
+from six import PY2
 
 import fs.copy
 from fs import open_fs
@@ -34,4 +40,298 @@ class TestCopy(unittest.TestCase):
 
         self.assertTrue(dst_fs.isdir('bar'))
         self.assertTrue(dst_fs.isdir('empty'))
-        self.assertTrue(dst_fs.isfile('bar/baz.txt'))
\ No newline at end of file
+        self.assertTrue(dst_fs.isfile('bar/baz.txt'))
+
+    def mkdirp(self, path):
+        #os.makedirs(path, exist_ok=True) only for python3.?
+        try:
+            os.makedirs(path)
+        except OSError as exc:
+            if exc.errno == errno.EEXIST and os.path.isdir(path):
+                pass
+            else:
+                raise
+
+    def _create_sandbox_dir(self, prefix='pyfilesystem2_sandbox_', home=None):
+        if home is None:
+            return tempfile.mkdtemp(prefix=prefix)
+        else:
+            sandbox_path = os.path.join(home, prefix)
+            self.mkdirp(sandbox_path)
+            return sandbox_path
+
+    def _touch(self, root, filepath):
+        #create abs filename
+        abs_filepath = os.path.join(root, filepath)
+        #create dir
+        dirname = os.path.dirname(abs_filepath)
+        self.mkdirp(dirname)
+        #touch file
+        with open(abs_filepath, 'a'):
+            os.utime(abs_filepath, None) #update the mtime in case the file exists, same as touch
+
+        return abs_filepath
+
+    def _write_file(self, filepath, write_chars=1024):
+        with open(filepath, 'w') as f:
+            f.write('1' * write_chars)
+        return filepath
+
+    def _delay_file_utime(self, filepath, delta_sec=None):
+        import calendar
+        from datetime import datetime
+        file_access_mod_time = int(calendar.timegm(datetime.now().timetuple())) + delta_sec
+        times = (file_access_mod_time, file_access_mod_time)
+        os.utime(filepath, times)
+
+    def test_copy_file_if_newer_dst_older(self):
+        try:
+            #create first dst ==> dst is older the src ==> file should be copied
+            dst_dir = self._create_sandbox_dir()
+            dst_file1 = self._touch(dst_dir, "file1.txt")
+            self._write_file(dst_file1)
+
+            src_dir = self._create_sandbox_dir()
+            src_file1 = self._touch(src_dir, "file1.txt")
+            self._write_file(src_file1)
+            #ensure src file is newer than dst, changing its modification time
+            self._delay_file_utime(src_file1, delta_sec=60)
+
+            src_fs = open_fs('osfs://' + src_dir)
+            dst_fs = open_fs('osfs://' + dst_dir)
+
+            self.assertTrue(dst_fs.exists("/file1.txt"))
+
+            copied = fs.copy.copy_file_if_newer(src_fs, "/file1.txt", dst_fs, "/file1.txt")
+
+            self.assertTrue(copied)
+            self.assertTrue(dst_fs.exists("/file1.txt"))
+        finally:
+            shutil.rmtree(src_dir)
+            shutil.rmtree(dst_dir)
+
+    def test_copy_file_if_newer_dst_doesnt_exists(self):
+        try:
+            src_dir = self._create_sandbox_dir()
+            src_file1 = self._touch(src_dir, "file1.txt")
+            self._write_file(src_file1)
+
+            dst_dir = self._create_sandbox_dir()
+
+            src_fs = open_fs('osfs://' + src_dir)
+            dst_fs = open_fs('osfs://' + dst_dir)
+
+
+            copied = fs.copy.copy_file_if_newer(src_fs, "/file1.txt", dst_fs, "/file1.txt")
+
+            self.assertTrue(copied)
+            self.assertTrue(dst_fs.exists("/file1.txt"))
+        finally:
+            shutil.rmtree(src_dir)
+            shutil.rmtree(dst_dir)
+
+    def test_copy_file_if_newer_dst_is_newer(self):
+        try:
+            src_dir = self._create_sandbox_dir()
+            src_file1 = self._touch(src_dir, "file1.txt")
+            self._write_file(src_file1)
+
+            dst_dir = self._create_sandbox_dir()
+            dst_file1 = self._touch(dst_dir, "file1.txt")
+            self._write_file(dst_file1)
+
+            src_fs = open_fs('osfs://' + src_dir)
+            dst_fs = open_fs('osfs://' + dst_dir)
+
+
+            self.assertTrue(dst_fs.exists("/file1.txt"))
+
+            copied = fs.copy.copy_file_if_newer(src_fs, "/file1.txt", dst_fs, "/file1.txt")
+
+            self.assertEqual(copied, False)
+        finally:
+            shutil.rmtree(src_dir)
+            shutil.rmtree(dst_dir)
+
+
+    def test_copy_fs_if_newer_dst_older(self):
+        try:
+            #create first dst ==> dst is older the src ==> file should be copied
+            dst_dir = self._create_sandbox_dir()
+            dst_file1 = self._touch(dst_dir, "file1.txt")
+            self._write_file(dst_file1)
+
+            src_dir = self._create_sandbox_dir()
+            src_file1 = self._touch(src_dir, "file1.txt")
+            self._write_file(src_file1)
+            #ensure src file is newer than dst, changing its modification time
+            self._delay_file_utime(src_file1, delta_sec=60)
+
+            src_fs = open_fs('osfs://' + src_dir)
+            dst_fs = open_fs('osfs://' + dst_dir)
+
+            self.assertTrue(dst_fs.exists("/file1.txt"))
+
+            copied = []
+            def on_copy(src_fs, src_path, dst_fs, dst_path):
+                copied.append(dst_path)
+
+            fs.copy.copy_fs_if_newer(src_fs, dst_fs, on_copy=on_copy)
+
+            self.assertEqual(copied, ["/file1.txt"])
+            self.assertTrue(dst_fs.exists("/file1.txt"))
+
+            src_fs.close()
+            dst_fs.close()
+
+        finally:
+            shutil.rmtree(src_dir)
+            shutil.rmtree(dst_dir)
+
+    def test_copy_fs_if_newer_when_dst_doesnt_exists(self):
+        try:
+            src_dir = self._create_sandbox_dir()
+            src_file1 = self._touch(src_dir, "file1.txt")
+            self._write_file(src_file1)
+
+            src_file2 = self._touch(src_dir, "one_level_down" + os.sep + "file2.txt")
+            self._write_file(src_file2)
+
+            dst_dir = self._create_sandbox_dir()
+
+            src_fs = open_fs('osfs://' + src_dir)
+            dst_fs = open_fs('osfs://' + dst_dir)
+
+            copied = []
+            def on_copy(src_fs, src_path, dst_fs, dst_path):
+                copied.append(dst_path)
+
+            fs.copy.copy_fs_if_newer(src_fs, dst_fs, on_copy=on_copy)
+
+            self.assertEqual(copied, ["/file1.txt", "/one_level_down/file2.txt"])
+            self.assertTrue(dst_fs.exists("/file1.txt"))
+            self.assertTrue(dst_fs.exists("/one_level_down/file2.txt"))
+
+            src_fs.close()
+            dst_fs.close()
+
+        finally:
+            shutil.rmtree(src_dir)
+            shutil.rmtree(dst_dir)
+
+    def test_copy_fs_if_newer_dont_copy_when_dst_exists(self):
+        try:
+            #src is older than dst => no copy should be necessary
+            src_dir = self._create_sandbox_dir()
+            src_file1 = self._touch(src_dir, "file1.txt")
+            self._write_file(src_file1)
+
+            dst_dir = self._create_sandbox_dir()
+            dst_file1 = self._touch(dst_dir, "file1.txt")
+            self._write_file(dst_file1)
+            #ensure dst file is newer than src, changing its modification time
+            self._delay_file_utime(dst_file1, delta_sec=60)
+
+            src_fs = open_fs('osfs://' + src_dir)
+            dst_fs = open_fs('osfs://' + dst_dir)
+
+            self.assertTrue(dst_fs.exists("/file1.txt"))
+
+            copied = []
+            def on_copy(src_fs, src_path, dst_fs, dst_path):
+                copied.append(dst_path)
+
+            fs.copy.copy_fs_if_newer(src_fs, dst_fs, on_copy=on_copy)
+
+            self.assertEqual(copied, [])
+            self.assertTrue(dst_fs.exists("/file1.txt"))
+
+            src_fs.close()
+            dst_fs.close()
+
+        finally:
+            shutil.rmtree(src_dir)
+            shutil.rmtree(dst_dir)
+
+    def test_copy_dir_if_newer_one_dst_doesnt_exist(self):
+        try:
+
+            src_dir = self._create_sandbox_dir()
+            src_file1 = self._touch(src_dir, "file1.txt")
+            self._write_file(src_file1)
+
+            src_file2 = self._touch(src_dir, "one_level_down" + os.sep + "file2.txt")
+            self._write_file(src_file2)
+
+            dst_dir = self._create_sandbox_dir()
+            dst_file1 = self._touch(dst_dir, "file1.txt")
+            self._write_file(dst_file1)
+            #ensure dst file is newer than src, changing its modification time
+            self._delay_file_utime(dst_file1, delta_sec=60)
+
+            src_fs = open_fs('osfs://' + src_dir)
+            dst_fs = open_fs('osfs://' + dst_dir)
+
+            copied = []
+            def on_copy(src_fs, src_path, dst_fs, dst_path):
+                copied.append(dst_path)
+
+            fs.copy.copy_dir_if_newer(src_fs, "/", dst_fs, "/", on_copy=on_copy)
+
+            self.assertEqual(copied, ["/one_level_down/file2.txt"])
+            self.assertTrue(dst_fs.exists("/one_level_down/file2.txt"))
+
+            src_fs.close()
+            dst_fs.close()
+        finally:
+            shutil.rmtree(src_dir)
+            shutil.rmtree(dst_dir)
+
+    def test_copy_dir_if_newer_same_fs(self):
+        try:
+            src_dir = self._create_sandbox_dir()
+            src_file1 = self._touch(src_dir, "src" + os.sep + "file1.txt")
+            self._write_file(src_file1)
+
+            dst_dir = self._create_sandbox_dir(home=src_dir)
+
+            src_fs = open_fs('osfs://' + src_dir)
+
+            copied = []
+            def on_copy(src_fs, src_path, dst_fs, dst_path):
+                copied.append(dst_path)
+
+            fs.copy.copy_dir_if_newer(src_fs, "/src", src_fs, "/dst", on_copy=on_copy)
+
+            self.assertEqual(copied, ["/dst/file1.txt"])
+            self.assertTrue(src_fs.exists("/dst/file1.txt"))
+
+            src_fs.close()
+
+        finally:
+            shutil.rmtree(src_dir)
+
+    def test_copy_dir_if_newer_multiple_files(self):
+
+        try:
+            src_dir = self._create_sandbox_dir()
+            src_fs = open_fs('osfs://' + src_dir)
+            src_fs.makedirs('foo/bar')
+            src_fs.makedirs('foo/empty')
+            src_fs.touch('test.txt')
+            src_fs.touch('foo/bar/baz.txt')
+
+            dst_dir = self._create_sandbox_dir()
+            dst_fs = open_fs('osfs://' + dst_dir)
+
+            fs.copy.copy_dir_if_newer(src_fs, '/foo', dst_fs, '/')
+
+            self.assertTrue(dst_fs.isdir('bar'))
+            self.assertTrue(dst_fs.isdir('empty'))
+            self.assertTrue(dst_fs.isfile('bar/baz.txt'))
+        finally:
+            shutil.rmtree(src_dir)
+            shutil.rmtree(dst_dir)
+
+if __name__ == "__main__":
+    unittest.main()
\ No newline at end of file