Repository: PyFilesystem/pyfilesystem2
Pull request #80 (2017-09-24 17:53:48)

Mirror
Adds 'mirroring' functionality.

coveralls: 
[![Coverage Status](https://coveralls.io/builds/13408292/badge)](https://coveralls.io/builds/13408292)

Coverage remained the same at 100.0% when pulling **4ce50015f546194e1819bb8a3c338645531f5aae on mirror** into **f8d2ce98352d2c199f499c1a2176f508ff69004a on master**.

coveralls: 
[![Coverage Status](https://coveralls.io/builds/13408332/badge)](https://coveralls.io/builds/13408332)

Coverage remained the same at 100.0% when pulling **a55745f08fa5f2323cd91ae27f3767824225726b on mirror** into **f8d2ce98352d2c199f499c1a2176f508ff69004a on master**.

coveralls: 
[![Coverage Status](https://coveralls.io/builds/13420642/badge)](https://coveralls.io/builds/13420642)

Coverage remained the same at 100.0% when pulling **fd1307a1fff682b096871fc427169cba89416fe2 on mirror** into **f8d2ce98352d2c199f499c1a2176f508ff69004a on master**.

coveralls: 
[![Coverage Status](https://coveralls.io/builds/13463891/badge)](https://coveralls.io/builds/13463891)

Coverage remained the same at 100.0% when pulling **47fd586063c415e0ae814547828e2c96d9a7c519 on mirror** into **f8d2ce98352d2c199f499c1a2176f508ff69004a on master**.

coveralls: 
[![Coverage Status](https://coveralls.io/builds/13463891/badge)](https://coveralls.io/builds/13463891)

Coverage remained the same at 100.0% when pulling **47fd586063c415e0ae814547828e2c96d9a7c519 on mirror** into **f8d2ce98352d2c199f499c1a2176f508ff69004a on master**.


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e246ea5..f3a3c46 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [2.0.11]
+
+### Added
+
+- fs.mirror
+
 ## [2.0.10]
 
 ### Added
diff --git a/docs/source/guide.rst b/docs/source/guide.rst
index b964779..e6500a5 100644
--- a/docs/source/guide.rst
+++ b/docs/source/guide.rst
@@ -221,4 +221,4 @@ The :func:`~fs.copy.copy_fs` and :func:`~fs.copy.copy_dir` functions also accept
     >>> from fs.walk import Walker
     >>> copy_fs('~/projects', 'zip://projects.zip', walker=Walker(filter=['*.py']))
 
-
+An alternative to copying is *mirroring*, which will copy a filesystem them keep it up to date by copying only changed files / directories. See :func:`~fs.mirror.mirror`.
diff --git a/docs/source/reference.rst b/docs/source/reference.rst
index fd8cfd4..752c598 100644
--- a/docs/source/reference.rst
+++ b/docs/source/reference.rst
@@ -10,6 +10,7 @@ Reference
    reference/enums.rst
    reference/errors.rst
    reference/info_objects.rst
+   reference/mirror.rst
    reference/move.rst
    reference/mode.rst
    reference/opener.rst
diff --git a/docs/source/reference/mirror.rst b/docs/source/reference/mirror.rst
new file mode 100644
index 0000000..db03cd3
--- /dev/null
+++ b/docs/source/reference/mirror.rst
@@ -0,0 +1,20 @@
+fs.mirror
+=========
+
+Create a duplicate of a filesystem.
+
+Mirroring will create a copy of a source filesystem on a destination
+filesystem. If there are no files on the destination, then mirroring
+is simply a straight copy. If there are any files or directories on the
+destination they may be deleted or modified to match the source.
+
+In order to avoid redundant copying of files, ``mirror`` can compare
+timestamps, and only copy files with a newer modified date. This
+timestamp comparison is only done if the file sizes are different.
+
+This scheme will work if you have mirrored a directory previously, and
+you would like to copy any changes. Otherwise you should set the
+``copy_if_newer`` parameter to ``False`` to guarantee an exact copy, at
+the expense of potentially copying extra files.
+
+.. autofunction:: fs.mirror.mirror
diff --git a/fs/_version.py b/fs/_version.py
index 44deabc..c54e9cf 100644
--- a/fs/_version.py
+++ b/fs/_version.py
@@ -1,2 +1,2 @@
 """Version, used in module and setup.py."""
-__version__ = "2.0.10a2"
+__version__ = "2.0.11"
diff --git a/fs/base.py b/fs/base.py
index 5573671..3314568 100644
--- a/fs/base.py
+++ b/fs/base.py
@@ -1,6 +1,6 @@
 """
 fs.base
-========
+=======
 
 PyFilesystem base class
 
diff --git a/fs/copy.py b/fs/copy.py
index 008d263..5d11f96 100644
--- a/fs/copy.py
+++ b/fs/copy.py
@@ -89,6 +89,7 @@ def _source_is_newer(src_fs, src_path, dst_fs, dst_path):
         #todo: should log something here
         return True
 
+
 def copy_file(src_fs, src_path, dst_fs, dst_path):
     """
     Copy a file from one filesystem to another. If the destination
diff --git a/fs/mirror.py b/fs/mirror.py
new file mode 100644
index 0000000..9594029
--- /dev/null
+++ b/fs/mirror.py
@@ -0,0 +1,107 @@
+"""
+
+Create a 'mirror' of a filesystem.
+
+"""
+
+from __future__ import print_function
+from __future__ import unicode_literals
+
+
+from .copy import copy_file
+from .errors import ResourceNotFound
+from .walk import Walker
+from .opener import manage_fs
+
+
+def _compare(info1, info2):
+    """
+    Compare two (file) info objects and return True if the file should
+    be copied, or False if they should not.
+
+    """
+    # Check filesize has changed
+    if info1.size != info2.size:
+        return True
+    # Check modified dates
+    date1 = info1.modified
+    date2 = info2.modified
+    return date1 is None or date2 is None or date1 > date2
+
+
+def mirror(src_fs, dst_fs, walker=None, copy_if_newer=True):
+    """
+    Mirror files / directories from one filesystem to another.
+
+    :param src_fs: A source filesystem.
+    :type src_fs: FS URL or instance.
+    :param dst_fs: A destination filesystem.
+    :type dst_fs: FS URL or instance.
+    :param walker: An optional waler instance.
+    :type walker: :class:`~fs.walk.Walker`
+    :param bool copy_if_newer: Only copy newer files.
+
+    Mirroring a filesystem will create an exact copy of ``src_fs`` on
+    ``dst_fs``, by removing any files / directories on the destination
+    that aren't on the source, and copying files that aren't.
+
+    """
+    with manage_fs(src_fs, writeable=False) as _src_fs:
+        with manage_fs(dst_fs, create=True) as _dst_fs:
+            with _src_fs.lock(), _dst_fs.lock():
+                return _mirror(
+                    _src_fs,
+                    _dst_fs,
+                    walker=walker,
+                    copy_if_newer=copy_if_newer
+                )
+
+
+def _mirror(src_fs, dst_fs, walker=None, copy_if_newer=True):
+    walker = walker or Walker()
+    walk = walker.walk(src_fs, namespaces=['details'])
+    for path, dirs, files in walk:
+        try:
+            dst = {
+                info.name: info
+                for info in dst_fs.scandir(path, namespaces=['details'])
+            }
+        except ResourceNotFound:
+            dst_fs.makedir(path)
+            dst = {}
+
+        # Copy files
+        for _file in files:
+            _path = _file.make_path(path)
+            dst_file = dst.pop(_file.name, None)
+            if dst_file is not None:
+                if dst_file.is_dir:
+                    # Destination is a directory, remove it
+                    dst_fs.removetree(_path)
+                else:
+                    # Compare file info
+                    if copy_if_newer and not _compare(_file, dst_file):
+                        continue
+            copy_file(src_fs, _path, dst_fs, _path)
+
+        # Make directories
+        for _dir in dirs:
+            _path = _dir.make_path(path)
+            dst_dir = dst.pop(_dir.name, None)
+            if dst_dir is not None:
+                # Directory name exists on dst
+                if not dst_dir.is_dir:
+                    # Not a directory, so remove it
+                    dst_fs.remove(_path)
+            else:
+                # Make the directory in dst
+                dst_fs.makedir(_path, recreate=True)
+
+        # Remove any remaining resources
+        while dst:
+            _, info = dst.popitem()
+            _path = info.make_path(path)
+            if info.is_dir:
+                dst_fs.removetree(_path)
+            else:
+                dst_fs.remove(_path)


Test patch:
diff --git a/tests/test_mirror.py b/tests/test_mirror.py
new file mode 100644
index 0000000..662093f
--- /dev/null
+++ b/tests/test_mirror.py
@@ -0,0 +1,104 @@
+from __future__ import unicode_literals
+
+import unittest
+
+from fs.mirror import mirror
+from fs import open_fs
+
+
+class TestMirror(unittest.TestCase):
+
+
+    def _contents(self, fs):
+        """Extract an FS in to a simple data structure."""
+        contents = []
+        for path, dirs, files in fs.walk():
+            for info in dirs:
+                _path = info.make_path(path)
+                contents.append((_path, 'dir', b''))
+            for info in files:
+                _path = info.make_path(path)
+                contents.append((_path, 'file', fs.getbytes(_path)))
+        return sorted(contents)
+
+    def assert_compare_fs(self, fs1, fs2):
+        """Assert filesystems and contents are the same."""
+        self.assertEqual(
+            self._contents(fs1),
+            self._contents(fs2)
+        )
+
+    def test_empty_mirror(self):
+        m1 = open_fs('mem://')
+        m2 = open_fs('mem://')
+        mirror(m1, m2)
+        self.assert_compare_fs(m1, m2)
+
+    def test_mirror_one_file(self):
+        m1 = open_fs('mem://')
+        m1.settext('foo', 'hello')
+        m2 = open_fs('mem://')
+        mirror(m1, m2)
+        self.assert_compare_fs(m1, m2)
+
+    def test_mirror_one_file_one_dir(self):
+        m1 = open_fs('mem://')
+        m1.settext('foo', 'hello')
+        m1.makedir('bar')
+        m2 = open_fs('mem://')
+        mirror(m1, m2)
+        self.assert_compare_fs(m1, m2)
+
+    def test_mirror_delete_replace(self):
+        m1 = open_fs('mem://')
+        m1.settext('foo', 'hello')
+        m1.makedir('bar')
+        m2 = open_fs('mem://')
+        mirror(m1, m2)
+        self.assert_compare_fs(m1, m2)
+        m2.remove('foo')
+        mirror(m1, m2)
+        self.assert_compare_fs(m1, m2)
+        m2.removedir('bar')
+        mirror(m1, m2)
+        self.assert_compare_fs(m1, m2)
+
+    def test_mirror_extra_dir(self):
+        m1 = open_fs('mem://')
+        m1.settext('foo', 'hello')
+        m1.makedir('bar')
+        m2 = open_fs('mem://')
+        m2.makedir('baz')
+        mirror(m1, m2)
+        self.assert_compare_fs(m1, m2)
+
+    def test_mirror_extra_file(self):
+        m1 = open_fs('mem://')
+        m1.settext('foo', 'hello')
+        m1.makedir('bar')
+        m2 = open_fs('mem://')
+        m2.makedir('baz')
+        m2.touch('egg')
+        mirror(m1, m2)
+        self.assert_compare_fs(m1, m2)
+
+    def test_mirror_wrong_type(self):
+        m1 = open_fs('mem://')
+        m1.settext('foo', 'hello')
+        m1.makedir('bar')
+        m2 = open_fs('mem://')
+        m2.makedir('foo')
+        m2.touch('bar')
+        mirror(m1, m2)
+        self.assert_compare_fs(m1, m2)
+
+    def test_mirror_update(self):
+        m1 = open_fs('mem://')
+        m1.settext('foo', 'hello')
+        m1.makedir('bar')
+        m2 = open_fs('mem://')
+        mirror(m1, m2)
+        self.assert_compare_fs(m1, m2)
+        m2.appendtext('foo', ' world!')
+        mirror(m1, m2)
+        self.assert_compare_fs(m1, m2)