Repository: Backblaze/B2_Command_Line_Tool

Issue:
--excludeDirRegex does not work when source is B2
The new filtering that lets you exclude an entire directory works in the `LocalFolder` class, but not the `B2Folder` class.

I think there are two possible approaches to fixing it: (1) change B2Folder to simulate the existence of directories, and check them for exclusion, or (2) extend `ScanPoliciesManager.should_exclude_file` to also test whether any of the directories in the path are excluded.  I like #2, but I think it would need optimization to avoid checking every parent directory of every file.

Discussion:

Patch:
diff --git a/b2/sync/scan_policies.py b/b2/sync/scan_policies.py
index 198c079..dfb9413 100644
--- a/b2/sync/scan_policies.py
+++ b/b2/sync/scan_policies.py
@@ -27,10 +27,45 @@ class RegexSet(object):
         return any(c.match(s) is not None for c in self._compiled_list)
 
 
+def convert_dir_regex_to_dir_prefix_regex(dir_regex):
+    """
+    The patterns used to match directory names (and file names) are allowed
+    to match a prefix of the name.  This 'feature' was unintentional, but is
+    being retained for compatibility.
+
+    This means that a regex that matches a directory name can't be used directly
+    to match against a file name and test whether the file should be excluded
+    because it matches the directory.
+
+    The pattern 'photos' will match directory names 'photos' and 'photos2',
+    and should exclude files 'photos/kitten.jpg', and 'photos2/puppy.jpg'.
+    It should not exclude 'photos.txt', because there is no directory name
+    that matches.
+
+    On the other hand, the pattern 'photos$' should match 'photos/kitten.jpg',
+    but not 'photos2/puppy.jpg', nor 'photos.txt'
+
+    If the original regex is valid, there are only two cases to consider:
+    either the regex ends in '$' or does not.
+    """
+    if dir_regex.endswith('$'):
+        return dir_regex[:-1] + r'/'
+    else:
+        return dir_regex + r'.*?/'
+
+
 class ScanPoliciesManager(object):
     """
     Policy object used when scanning folders for syncing, used to decide
     which files to include in the list of files to be synced.
+
+    Code that scans through files should at least use should_exclude_file()
+    to decide whether each file should be included; it will check include/exclude
+    patterns for file names, as well as patterns for excluding directeries.
+
+    Code that scans may optionally use should_exclude_directory() to test whether
+    it can skip a directory completely and not bother listing the files and
+    sub-directories in it.
     """
 
     def __init__(
@@ -40,6 +75,9 @@ class ScanPoliciesManager(object):
         include_file_regexes=tuple(),
     ):
         self._exclude_dir_set = RegexSet(exclude_dir_regexes)
+        self._exclude_file_because_of_dir_set = RegexSet(
+            map(convert_dir_regex_to_dir_prefix_regex, exclude_dir_regexes)
+        )
         self._exclude_file_set = RegexSet(exclude_file_regexes)
         self._include_file_set = RegexSet(include_file_regexes)
 
@@ -51,8 +89,12 @@ class ScanPoliciesManager(object):
                           being scanned.
         :return: True iff excluded.
         """
-        return self._exclude_file_set.matches(file_path) and \
-               not self._include_file_set.matches(file_path)
+        exclude_because_of_dir = self._exclude_file_because_of_dir_set.matches(file_path)
+        exclude_because_of_file = (
+            self._exclude_file_set.matches(file_path) and
+            not self._include_file_set.matches(file_path)
+        )
+        return exclude_because_of_dir or exclude_because_of_file
 
     def should_exclude_directory(self, dir_path):
         """


Test patch:
diff --git a/test/test_scan_policies.py b/test/test_scan_policies.py
index f3bb797..853730d 100644
--- a/test/test_scan_policies.py
+++ b/test/test_scan_policies.py
@@ -30,8 +30,20 @@ class TestScanPolicies(TestBase):
 
     def test_exclude_dir(self):
         policy = ScanPoliciesManager(
-            include_file_regexes=['.*[.]txt$'], exclude_dir_regexes=['alfa$']
+            include_file_regexes=['.*[.]txt$'], exclude_dir_regexes=['alfa', 'bravo$']
         )
         self.assertTrue(policy.should_exclude_directory('alfa'))
-        self.assertFalse(policy.should_exclude_directory('alfa2'))
-        self.assertFalse(policy.should_exclude_directory('alfa/hello'))
+        self.assertTrue(policy.should_exclude_directory('alfa2'))
+        self.assertTrue(policy.should_exclude_directory('alfa/hello'))
+
+        self.assertTrue(policy.should_exclude_directory('bravo'))
+        self.assertFalse(policy.should_exclude_directory('bravo2'))
+        self.assertFalse(policy.should_exclude_directory('bravo/hello'))
+
+        self.assertTrue(policy.should_exclude_file('alfa/foo'))
+        self.assertTrue(policy.should_exclude_file('alfa2/hello/foo'))
+        self.assertTrue(policy.should_exclude_file('alfa/hello/foo.txt'))
+
+        self.assertTrue(policy.should_exclude_file('bravo/foo'))
+        self.assertFalse(policy.should_exclude_file('bravo2/hello/foo'))
+        self.assertTrue(policy.should_exclude_file('bravo/hello/foo.txt'))