Repository: PyFilesystem/pyfilesystem2
Pull request #265 (2019-02-15 11:05:36)

Walk filter
## Type of changes

- [ ] Bug fix
- [X] New feature
- [ ] Documentation / docstrings
- [ ] Tests
- [ ] Other

## Checklist

- [X] I've run the latest [black](https://github.com/ambv/black) with default args on new code.
- [X] I've updated CHANGELOG.md and CONTRIBUTORS.md where appropriate.
- [X] I've added tests for new code.
- [X] I accept that @willmcgugan may be pedantic in the code review.

## Description

Added an `exclude` argument to the walker, which excludes files matching any of the given patterns. This can be useful to ignore temporary or hidden files.

Also added `filter_dirs` argument to the walker, which will only recurse those directories which match any of the given filters. This one was mainly added for parity with other filter / exclude arguments. I can't actually think of a use case for this functionality!


coveralls: 
[![Coverage Status](https://coveralls.io/builds/21665258/badge)](https://coveralls.io/builds/21665258)

Coverage increased (+0.0004%) to 99.696% when pulling **be03e52fe4011d10b06d0832bd71937c418797f9 on walk-filter** into **e422bc0032fadf55806b7901581324064f2db774 on master**.


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bdb724e..a5b9e29 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [2.4.0] - 2019-02-15
+
+### Added
+
+- Added `exclude` and `filter_dirs` arguments to walk
+- Micro-optimizations to walk
+
 ## [2.3.1] - 2019-02-10
 
 ### Fixed
diff --git a/fs/_version.py b/fs/_version.py
index 7af6079..8f94831 100644
--- a/fs/_version.py
+++ b/fs/_version.py
@@ -1,3 +1,3 @@
 """Version, used in module and setup.py.
 """
-__version__ = "2.3.1"
+__version__ = "2.4.0"
diff --git a/fs/copy.py b/fs/copy.py
index e857707..9b171d3 100644
--- a/fs/copy.py
+++ b/fs/copy.py
@@ -35,7 +35,7 @@ def copy_fs(
         walker (~fs.walk.Walker, optional): A walker object that will be
             used to scan for files in ``src_fs``. Set this if you only want
             to consider a sub-set of the resources in ``src_fs``.
-        on_copy (callable):A function callback called after a single file copy
+        on_copy (callable): A function callback called after a single file copy
             is executed. Expected signature is ``(src_fs, src_path, dst_fs,
             dst_path)``.
         workers (int): Use `worker` threads to copy data, or ``0`` (default) for
diff --git a/fs/walk.py b/fs/walk.py
index 5e8483d..36ef844 100644
--- a/fs/walk.py
+++ b/fs/walk.py
@@ -17,7 +17,7 @@ import six
 from ._repr import make_repr
 from .errors import FSError
 from .path import abspath
-from .path import join
+from .path import combine
 from .path import normpath
 
 if False:  # typing.TYPE_CHECKING
@@ -68,6 +68,12 @@ class Walker(object):
             a list of filename patterns, e.g. ``['*.py']``. Files will
             only be returned if the final component matches one of the
             patterns.
+        exclude (list, optional): If supplied, this parameter should be
+            a list of filename patterns, e.g. ``['~*']``. Files matching
+            any of these patterns will be removed from the walk.
+        filter_dirs (list, optional): A list of patterns that will be used
+            to match directories paths. The walk will only open directories
+            that match at least one of these patterns.
         exclude_dirs (list, optional): A list of patterns that will be
             used to filter out directories from the walk. e.g.
             ``['*.svn', '*.git']``.
@@ -81,6 +87,8 @@ class Walker(object):
         on_error=None,  # type: Optional[OnError]
         search="breadth",  # type: Text
         filter=None,  # type: Optional[List[Text]]
+        exclude=None,  # type: Optional[List[Text]]
+        filter_dirs=None,  # type: Optional[List[Text]]
         exclude_dirs=None,  # type: Optional[List[Text]]
         max_depth=None,  # type: Optional[int]
     ):
@@ -99,6 +107,8 @@ class Walker(object):
         self.on_error = on_error
         self.search = search
         self.filter = filter
+        self.exclude = exclude
+        self.filter_dirs = filter_dirs
         self.exclude_dirs = exclude_dirs
         self.max_depth = max_depth
         super(Walker, self).__init__()
@@ -169,6 +179,8 @@ class Walker(object):
             on_error=(self.on_error, None),
             search=(self.search, "breadth"),
             filter=(self.filter, None),
+            exclude=(self.exclude, None),
+            filter_dirs=(self.filter_dirs, None),
             exclude_dirs=(self.exclude_dirs, None),
             max_depth=(self.max_depth, None),
         )
@@ -192,6 +204,8 @@ class Walker(object):
         """
         if self.exclude_dirs is not None and fs.match(self.exclude_dirs, info.name):
             return False
+        if self.filter_dirs is not None and not fs.match(self.filter_dirs, info.name):
+            return False
         return self.check_open_dir(fs, path, info)
 
     def check_open_dir(self, fs, path, info):
@@ -251,6 +265,9 @@ class Walker(object):
             bool: `True` if the file should be included.
 
         """
+
+        if self.exclude is not None and fs.match(self.exclude, info.name):
+            return False
         return fs.match(self.filter, info.name)
 
     def _scan(
@@ -341,9 +358,10 @@ class Walker(object):
             recursively within the given directory.
 
         """
+        _combine = combine
         for _path, info in self._iter_walk(fs, path=path):
             if info is not None and not info.is_dir:
-                yield join(_path, info.name)
+                yield _combine(_path, info.name)
 
     def dirs(self, fs, path="/"):
         # type: (FS, Text) -> Iterator[Text]
@@ -358,9 +376,10 @@ class Walker(object):
             recursively within the given directory.
 
         """
+        _combine = combine
         for _path, info in self._iter_walk(fs, path=path):
             if info is not None and info.is_dir:
-                yield join(_path, info.name)
+                yield _combine(_path, info.name)
 
     def info(
         self,
@@ -381,10 +400,11 @@ class Walker(object):
             (str, Info): a tuple of ``(<absolute path>, <resource info>)``.
 
         """
+        _combine = combine
         _walk = self._iter_walk(fs, path=path, namespaces=namespaces)
         for _path, info in _walk:
             if info is not None:
-                yield join(_path, info.name), info
+                yield _combine(_path, info.name), info
 
     def _walk_breadth(
         self,
@@ -398,19 +418,27 @@ class Walker(object):
         queue = deque([path])
         push = queue.appendleft
         pop = queue.pop
-        depth = self._calculate_depth(path)
+
+        _combine = combine
+        _scan = self._scan
+        _calculate_depth = self._calculate_depth
+        _check_open_dir = self._check_open_dir
+        _check_scan_dir = self._check_scan_dir
+        _check_file = self.check_file
+
+        depth = _calculate_depth(path)
 
         while queue:
             dir_path = pop()
-            for info in self._scan(fs, dir_path, namespaces=namespaces):
+            for info in _scan(fs, dir_path, namespaces=namespaces):
                 if info.is_dir:
-                    _depth = self._calculate_depth(dir_path) - depth + 1
-                    if self._check_open_dir(fs, dir_path, info):
+                    _depth = _calculate_depth(dir_path) - depth + 1
+                    if _check_open_dir(fs, dir_path, info):
                         yield dir_path, info  # Opened a directory
-                        if self._check_scan_dir(fs, dir_path, info, _depth):
-                            push(join(dir_path, info.name))
+                        if _check_scan_dir(fs, dir_path, info, _depth):
+                            push(_combine(dir_path, info.name))
                 else:
-                    if self.check_file(fs, info):
+                    if _check_file(fs, info):
                         yield dir_path, info  # Found a file
             yield dir_path, None  # End of directory
 
@@ -425,15 +453,18 @@ class Walker(object):
         """
         # No recursion!
 
-        def scan(path):
-            # type: (Text) -> Iterator[Info]
-            """Perform scan."""
-            return self._scan(fs, path, namespaces=namespaces)
+        _combine = combine
+        _scan = self._scan
+        _calculate_depth = self._calculate_depth
+        _check_open_dir = self._check_open_dir
+        _check_scan_dir = self._check_scan_dir
+        _check_file = self.check_file
+        depth = _calculate_depth(path)
 
         stack = [
-            (path, scan(path), None)
+            (path, _scan(fs, path, namespaces=namespaces), None)
         ]  # type: List[Tuple[Text, Iterator[Info], Optional[Tuple[Text, Info]]]]
-        depth = self._calculate_depth(path)
+
         push = stack.append
 
         while stack:
@@ -445,15 +476,21 @@ class Walker(object):
                 yield dir_path, None
                 del stack[-1]
             elif info.is_dir:
-                _depth = self._calculate_depth(dir_path) - depth + 1
-                if self._check_open_dir(fs, dir_path, info):
-                    if self._check_scan_dir(fs, dir_path, info, _depth):
-                        _path = join(dir_path, info.name)
-                        push((_path, scan(_path), (dir_path, info)))
+                _depth = _calculate_depth(dir_path) - depth + 1
+                if _check_open_dir(fs, dir_path, info):
+                    if _check_scan_dir(fs, dir_path, info, _depth):
+                        _path = _combine(dir_path, info.name)
+                        push(
+                            (
+                                _path,
+                                _scan(fs, _path, namespaces=namespaces),
+                                (dir_path, info),
+                            )
+                        )
                     else:
                         yield dir_path, info
             else:
-                if self.check_file(fs, info):
+                if _check_file(fs, info):
                     yield dir_path, info
 
 
@@ -525,6 +562,12 @@ class BoundWalker(typing.Generic[_F]):
                 of file name patterns, e.g. ``['*.py']``. Files will only be
                 returned if the final component matches one of the
                 patterns.
+            exclude (list, optional): If supplied, this parameter should be
+                a list of filename patterns, e.g. ``['~*', '.*']``. Files matching
+                any of these patterns will be removed from the walk.
+            filter_dirs (list, optional): A list of patterns that will be used
+                to match directories paths. The walk will only open directories
+                that match at least one of these patterns.
             exclude_dirs (list): A list of patterns that will be used
                 to filter out directories from the walk, e.g. ``['*.svn',
                 '*.git']``.
@@ -574,6 +617,12 @@ class BoundWalker(typing.Generic[_F]):
                 of file name patterns, e.g. ``['*.py']``. Files will only be
                 returned if the final component matches one of the
                 patterns.
+            exclude (list, optional): If supplied, this parameter should be
+                a list of filename patterns, e.g. ``['~*', '.*']``. Files matching
+                any of these patterns will be removed from the walk.
+            filter_dirs (list, optional): A list of patterns that will be used
+                to match directories paths. The walk will only open directories
+                that match at least one of these patterns.
             exclude_dirs (list): A list of patterns that will be used
                 to filter out directories from the walk, e.g. ``['*.svn',
                 '*.git']``.
@@ -606,6 +655,9 @@ class BoundWalker(typing.Generic[_F]):
                 `False` to re-raise it.
             search (str): If ``'breadth'`` then the directory will be
                 walked *top down*. Set to ``'depth'`` to walk *bottom up*.
+            filter_dirs (list, optional): A list of patterns that will be used
+                to match directories paths. The walk will only open directories
+                that match at least one of these patterns.
             exclude_dirs (list): A list of patterns that will be used
                 to filter out directories from the walk, e.g. ``['*.svn',
                 '*.git']``.
@@ -650,6 +702,12 @@ class BoundWalker(typing.Generic[_F]):
                 of file name patterns, e.g. ``['*.py']``. Files will only be
                 returned if the final component matches one of the
                 patterns.
+            exclude (list, optional): If supplied, this parameter should be
+                a list of filename patterns, e.g. ``['~*', '.*']``. Files matching
+                any of these patterns will be removed from the walk.
+            filter_dirs (list, optional): A list of patterns that will be used
+                to match directories paths. The walk will only open directories
+                that match at least one of these patterns.
             exclude_dirs (list): A list of patterns that will be used
                 to filter out directories from the walk, e.g. ``['*.svn',
                 '*.git']``.


Test patch:
diff --git a/tests/test_walk.py b/tests/test_walk.py
index 23d2d7f..8bf57dc 100644
--- a/tests/test_walk.py
+++ b/tests/test_walk.py
@@ -63,6 +63,22 @@ class TestWalk(unittest.TestCase):
         ]
         self.assertEqual(_walk, expected)
 
+    def test_walk_filter_dirs(self):
+        _walk = []
+        for step in self.fs.walk(filter_dirs=["foo*"]):
+            self.assertIsInstance(step, walk.Step)
+            path, dirs, files = step
+            _walk.append(
+                (path, [info.name for info in dirs], [info.name for info in files])
+            )
+        expected = [
+            ("/", ["foo1", "foo2", "foo3"], []),
+            ("/foo1", [], ["top1.txt", "top2.txt"]),
+            ("/foo2", [], ["top3.bin"]),
+            ("/foo3", [], []),
+        ]
+        self.assertEqual(_walk, expected)
+
     def test_walk_depth(self):
         _walk = []
         for step in self.fs.walk(search="depth"):
@@ -193,6 +209,19 @@ class TestWalk(unittest.TestCase):
 
         self.assertEqual(files, [])
 
+    def test_walk_files_exclude(self):
+        # Test exclude argument works
+        files = list(self.fs.walk.files(exclude=["*.txt"]))
+        self.assertEqual(files, ["/foo2/top3.bin"])
+
+        # Test exclude doesn't break filter
+        files = list(self.fs.walk.files(filter=["*.bin"], exclude=["*.txt"]))
+        self.assertEqual(files, ["/foo2/top3.bin"])
+
+        # Test excluding everything
+        files = list(self.fs.walk.files(exclude=["*"]))
+        self.assertEqual(files, [])
+
     def test_walk_info(self):
         walk = []
         for path, info in self.fs.walk.info():