Repository: facelessuser/wcmatch (python)

Issue:
Possibly force or add a flag to force directories to be returned with trailing slash (glob)
`globmatch`  (when `REALPATH` is not enabled) only recognizes a path as being a directory if the filename ends with a trailing slash. `glob` only really returns a path with a slash when knows the file is a directory because it explicitly checked. This is similar to how bash can at times return a directory with or without a trailing slash depending on the pattern it is given. It may be useful to add a flag to force a directory to be returned with a slash. This would require additional `stat` calls in places we may not be calling `stat`. For this reason, I think this should be optional.

Pull request:
Add `MARK` flag to mark directories with trailing slashes
Closes #44 

Patch:
diff --git a/docs/src/markdown/changelog.md b/docs/src/markdown/changelog.md
index 92de27d..8f1125c 100644
--- a/docs/src/markdown/changelog.md
+++ b/docs/src/markdown/changelog.md
@@ -3,6 +3,7 @@
 ## 3.1.0
 
 - **NEW**: Deprecated `WcMatch` class methods `kill` and `reset`. `WcMatch` should be broken with a simple `break` statement instead.
+- **NEW**: Add a new flag `MARK` to force `glob` to return directories with a trailing slash.
 
 ## 3.0.2
 
diff --git a/docs/src/markdown/glob.md b/docs/src/markdown/glob.md
index 5a5b22d..4330942 100644
--- a/docs/src/markdown/glob.md
+++ b/docs/src/markdown/glob.md
@@ -351,6 +351,26 @@ True
 True
 ```
 
+#### `glob.MARK, glob.MK` {: #globmark}
+
+`MARK` ensures that [`glob`](#globglob) returns all directories with a trailing slash. This makes it very clear which
+paths are directories and allows you to save calling `os.path.isdir` as you can simply check for a path separator at the
+end of the path. This flag only applies to calls to `glob` or `iglob`.
+
+If you are passing the returned files from `glob` to [`globfilter`](#globglobfilter) or [`globmatch`](#globglobmatch),
+it is important to ensure directory paths have trailing slashes as these functions have no way of telling the path is a
+directory otherwise (except when [`REALPATH`](#globrealpath) is enabled). If you have `REALPATH` enabled, ensuring the
+files have trailing slashes can still save you a call to `os.path.isdir` as `REALPATH` resorts to calling it if there is
+no trailing slash.
+
+```pycon3
+>>> from wcmatch import glob
+>>> glob.glob('*', flags=glob.MARK)
+['appveyor.yml', 'base.patch', 'basematch.diff', 'docs\\', 'LICENSE.md', 'MANIFEST.in', 'mkdocs.yml', 'README.md', 'requirements\\', 'setup.cfg', 'setup.py', 'tests\\', 'tools\\', 'tox.ini', 'wcmatch\\']
+>>> glob.glob('*')
+['appveyor.yml', 'base.patch', 'basematch.diff', 'docs', 'LICENSE.md', 'MANIFEST.in', 'mkdocs.yml', 'README.md', 'requirements', 'setup.cfg', 'setup.py', 'tests', 'tools', 'tox.ini', 'wcmatch']
+```
+
 --8<--
 refs.txt
 --8<--
diff --git a/wcmatch/_wcparse.py b/wcmatch/_wcparse.py
index 6b0a57a..c0f7628 100644
--- a/wcmatch/_wcparse.py
+++ b/wcmatch/_wcparse.py
@@ -1255,16 +1255,27 @@ class WcParse(object):
         return pattern
 
 
-def _fs_match(pattern, filename, sep, follow, symlinks):
+def _fs_match(pattern, filename, is_dir, sep, follow, symlinks):
     """
     Match path against the pattern.
 
     Since `globstar` doesn't match symlinks (unless `FOLLOW` is enabled), we must look for symlinks.
     If we identify a symlink in a `globstar` match, we know this result should not actually match.
+
+    We only check for the symlink if we know we are looking at a directory.
+    And we only call `lstat` if we can't find it in the cache.
+
+    We know it's a directory if:
+
+    1. If the base is a directory, all parts are directories.
+    2. If we are not the last part of the `globstar`, the part is a directory.
+    3. If the base is a file, but the part is not at the end, it is a directory.
+
     """
 
     matched = False
 
+    end = len(filename)
     base = None
     m = pattern.fullmatch(filename)
     if m:
@@ -1276,20 +1287,22 @@ def _fs_match(pattern, filename, sep, follow, symlinks):
             last = len(groups)
             for i, star in enumerate(m.groups(), 1):
                 if star:
+                    at_end = m.end(i) == end
                     parts = star.strip(sep).split(sep)
                     if base is None:
                         base = filename[:m.start(i)]
                     for part in parts:
                         base = os.path.join(base, part)
-                        is_link = symlinks.get(base, None)
-                        if is_link is not None:
-                            matched = not is_link
-                        elif i != last or os.path.isdir(base):
-                            is_link = os.path.islink(base)
-                            symlinks[base] = is_link
-                            matched = not is_link
-                        if not matched:
-                            break
+                        if is_dir or i != last or not at_end:
+                            is_link = symlinks.get(base, None)
+                            if is_link is not None:
+                                matched = not is_link
+                            else:
+                                is_link = os.path.islink(base)
+                                symlinks[base] = is_link
+                                matched = not is_link
+                            if not matched:
+                                break
                 if matched:
                     break
     return matched
@@ -1301,12 +1314,14 @@ def _match_real(filename, include, exclude, follow, symlinks):
     sep = '\\' if util.platform() == "windows" else '/'
     if isinstance(filename, bytes):
         sep = os.fsencode(sep)
-    if not filename.endswith(sep) and os.path.isdir(filename):
+    is_dir = filename.endswith(sep)
+    if not is_dir and os.path.isdir(filename):
+        is_dir = True
         filename += sep
 
     matched = False
     for pattern in include:
-        if _fs_match(pattern, filename, sep, follow, symlinks):
+        if _fs_match(pattern, filename, is_dir, sep, follow, symlinks):
             matched = True
             break
 
@@ -1314,7 +1329,7 @@ def _match_real(filename, include, exclude, follow, symlinks):
         matched = True
         if exclude:
             for pattern in exclude:
-                if _fs_match(pattern, filename, sep, follow, symlinks):
+                if _fs_match(pattern, filename, is_dir, sep, follow, symlinks):
                     matched = False
                     break
     return matched
diff --git a/wcmatch/glob.py b/wcmatch/glob.py
index a7de627..4c41998 100644
--- a/wcmatch/glob.py
+++ b/wcmatch/glob.py
@@ -51,6 +51,8 @@ B = BRACE = _wcparse.BRACE
 P = REALPATH = _wcparse.REALPATH
 L = FOLLOW = _wcparse.FOLLOW
 S = SPLIT = _wcparse.SPLIT
+MK = MARK = 0x10000
+
 
 FLAG_MASK = (
     FORCECASE |
@@ -86,6 +88,9 @@ class Glob(object):
     def __init__(self, pattern, flags=0):
         """Initialize the directory walker object."""
 
+        self.mark = bool(flags & MARK)
+        if self.mark:
+            flags ^= MARK
         self.flags = _flag_transform(flags | _wcparse.REALPATH) ^ _wcparse.REALPATH
         self.follow_links = bool(flags & FOLLOW)
         self.dot = bool(flags & DOTMATCH)
@@ -182,7 +187,7 @@ class Glob(object):
         if os.path.isdir(scandir) and matcher is not None:
             for special in self.specials:
                 if matcher(special):
-                    yield os.path.join(curdir, special)
+                    yield os.path.join(curdir, special), True
 
         try:
             if NO_SCANDIR_WORKAROUND:
@@ -206,7 +211,7 @@ class Glob(object):
                             if deep and not self.follow_links and is_link:
                                 continue
                             if (not dir_only or is_dir) and (matcher is None or matcher(f.name)):
-                                yield path
+                                yield path, is_dir
                             if deep and is_dir:
                                 yield from self._glob_dir(path, matcher, dir_only, deep)
                         except OSError:  # pragma: no cover
@@ -226,7 +231,7 @@ class Glob(object):
                     if deep and not self.follow_links and is_link:
                         continue
                     if (not dir_only or is_dir) and (matcher is None or matcher(f)):
-                        yield path
+                        yield path, is_dir
                     if deep and is_dir:
                         yield from self._glob_dir(path, matcher, dir_only, deep)
         except OSError:  # pragma: no cover
@@ -289,14 +294,14 @@ class Glob(object):
             # There is one quirk though with Bash, if `curdir` had magic before `**`, Bash
             # omits the trailing `/`. We don't worry about that.
             if globstar_end and curdir:
-                yield os.path.join(curdir, self.empty)
+                yield os.path.join(curdir, self.empty), True
 
             # Search
-            for path in self._glob_dir(curdir, matcher, dir_only, deep=True):
+            for path, is_dir in self._glob_dir(curdir, matcher, dir_only, deep=True):
                 if this:
                     yield from self._glob(path, this, rest[:])
                 else:
-                    yield path
+                    yield path, is_dir
 
         elif not dir_only:
             # Files: no need to recursively search at this point as we are done.
@@ -308,11 +313,11 @@ class Glob(object):
             # and feed the results back through with the next pattern.
             this = rest.pop(0) if rest else None
             matcher = self._get_matcher(target)
-            for path in self._glob_dir(curdir, matcher, True):
+            for path, is_dir in self._glob_dir(curdir, matcher, True):
                 if this:
                     yield from self._glob(path, this, rest[:])
                 else:
-                    yield path
+                    yield path, is_dir
 
     def _get_starting_paths(self, curdir):
         """
@@ -324,7 +329,7 @@ class Glob(object):
         the actual casing and then compare.
         """
 
-        results = [curdir]
+        results = [(curdir, True)]
 
         if not self._is_parent(curdir) and not self._is_this(curdir):
             fullpath = os.path.abspath(curdir)
@@ -332,10 +337,15 @@ class Glob(object):
             dirname = os.path.dirname(fullpath)
             if basename:
                 matcher = self._get_matcher(basename)
-                results = [os.path.basename(name) for name in self._glob_dir(dirname, matcher, self)]
+                results = [(os.path.basename(name), is_dir) for name, is_dir in self._glob_dir(dirname, matcher, self)]
 
         return results
 
+    def format_path(self, path, is_dir, dir_only):
+        """Format path."""
+
+        return os.path.join(path, self.empty) if dir_only or (self.mark and is_dir) else path
+
     def glob(self):
         """Starts off the glob iterator."""
 
@@ -365,38 +375,38 @@ class Glob(object):
                     # Make sure case matches, but running case insensitive
                     # on a case sensitive file system may return more than
                     # one starting location.
-                    results = [curdir] if this.is_drive else self._get_starting_paths(curdir)
+                    results = [(curdir, True)] if this.is_drive else self._get_starting_paths(curdir)
                     if not results:
                         if not dir_only:
                             # There is no directory with this name,
                             # but we have a file and no directory restriction
-                            yield curdir
+                            yield self.format_path(curdir, False, dir_only)
                         return
 
                     if this.dir_only:
                         # Glob these directories if they exists
-                        for start in results:
-                            if os.path.isdir(start):
+                        for start, is_dir in results:
+                            if is_dir:
                                 rest = pattern[1:]
                                 if rest:
                                     this = rest.pop(0)
-                                    for match in self._glob(curdir, this, rest):
+                                    for match, is_dir in self._glob(curdir, this, rest):
                                         if not self._is_excluded(match, dir_only):
-                                            yield os.path.join(match, self.empty) if dir_only else match
+                                            yield self.format_path(match, is_dir, dir_only)
                                 elif not self._is_excluded(curdir, dir_only):
-                                    yield os.path.join(curdir, self.empty) if dir_only else curdir
+                                    yield self.format_path(curdir, is_dir, dir_only)
                     else:
                         # Return the file(s) and finish.
-                        for start in results:
-                            if os.path.lexists(start) and not self._is_excluded(start, dir_only):
-                                yield os.path.join(start, self.empty) if dir_only else start
+                        for match, is_dir in results:
+                            if os.path.lexists(match) and not self._is_excluded(match, dir_only):
+                                yield self.format_path(match, is_dir, dir_only)
                 else:
                     # Path starts with a magic pattern, let's get globbing
                     rest = pattern[:]
                     this = rest.pop(0)
-                    for match in self._glob(curdir if not curdir == self.current else self.empty, this, rest):
+                    for match, is_dir in self._glob(curdir if not curdir == self.current else self.empty, this, rest):
                         if not self._is_excluded(match, dir_only):
-                            yield os.path.join(match, self.empty) if dir_only else match
+                            yield self.format_path(match, is_dir, dir_only)
 
 
 def iglob(patterns, *, flags=0):


Test patch:
diff --git a/tests/test_glob.py b/tests/test_glob.py
index 749cf4b..22d12da 100644
--- a/tests/test_glob.py
+++ b/tests/test_glob.py
@@ -145,6 +145,18 @@ class _TestGlob:
 
         return os.path.join(cls.tempdir, *parts)
 
+    @classmethod
+    def res_norm(cls, *parts, absolute=False, mark=False):
+        """Normalize results adding a trailing slash if mark flag is enabled."""
+
+        if not absolute:
+            path = os.path.join(cls.tempdir, *parts)
+        else:
+            path = os.path.join(*parts)
+        if mark and os.path.isdir(os.path.join(cls.tempdir, *parts)):
+            path = os.path.join(path, b'' if isinstance(path, bytes) else '')
+        return path
+
     @classmethod
     def globjoin(cls, *parts):
         """Joins glob path."""
@@ -315,17 +327,16 @@ class _TestGlob:
             pytest.skip("Skipped")
 
         pattern = case[0]
-        if not cls.absolute:
-            results = [cls.norm(*x) for x in case[1]] if case[1] is not None else None
-        else:
-            results = [os.path.join(*list(x)) for x in case[1]] if case[1] is not None else None
-        flags = cls.DEFAULT_FLAGS
 
+        flags = cls.DEFAULT_FLAGS
         if len(case) > 2:
             flags ^= case[2]
-
         negative = flags & glob.N
 
+        results = [
+            cls.res_norm(*x, absolute=cls.absolute, mark=flags & glob.MARK) for x in case[1]
+        ] if case[1] is not None else None
+
         print("PATTERN: ", pattern)
         print("FLAGS: ", bin(flags))
         print("NEGATIVE: ", bin(negative))
@@ -795,6 +806,12 @@ class Testglob(_TestGlob):
         self.eval_glob_cases(case)
 
 
+class TestGlobMarked(Testglob):
+    """Test glob marked."""
+
+    DEFAULT_FLAGS = glob.BRACE | glob.EXTGLOB | glob.GLOBSTAR | glob.FOLLOW | glob.MARK
+
+
 class TestCWD(_TestGlob):
     """Test files in the current working directory."""
 
@@ -864,6 +881,12 @@ class TestGlobCornerCase(_TestGlob):
         self.eval_glob_cases(case)
 
 
+class TestGlobCornerCaseMarked(Testglob):
+    """Test glob marked."""
+
+    DEFAULT_FLAGS = glob.BRACE | glob.EXTGLOB | glob.GLOBSTAR | glob.FOLLOW | glob.MARK
+
+
 class TestGlobEscapes(unittest.TestCase):
     """Test escaping."""
 
diff --git a/tests/test_globmatch.py b/tests/test_globmatch.py
index 0bc2a09..ea0ba5a 100644
--- a/tests/test_globmatch.py
+++ b/tests/test_globmatch.py
@@ -955,6 +955,50 @@ class TestGlobMatchSpecial(unittest.TestCase):
             )
         )
 
+    def test_glob_integrity_marked(self):
+        """`globmatch` must match what glob globs with marked directories."""
+
+        # Number of slashes is inconsequential
+        # Glob really looks at what is in between. Multiple slashes are the same as one separator.
+        # UNC mounts are special cases and it matters there.
+        self.assertTrue(
+            all(
+                [
+                    glob.globmatch(
+                        x, '**/docs/**', flags=self.flags | glob.MARK
+                    ) for x in glob.glob('**/docs/**', flags=self.flags | glob.MARK)
+                ]
+            )
+        )
+        self.assertTrue(
+            all(
+                [
+                    glob.globmatch(
+                        x, '**/docs/**|!**/*.md', flags=self.flags | glob.SPLIT | glob.MARK
+                    ) for x in glob.glob('**/docs/**|!**/*.md', flags=self.flags | glob.SPLIT | glob.MARK)
+                ]
+            )
+        )
+
+        self.assertTrue(
+            all(
+                [
+                    glob.globmatch(
+                        x, '!**/*.md', flags=self.flags | glob.SPLIT | glob.MARK
+                    ) for x in glob.glob('!**/*.md', flags=self.flags | glob.SPLIT | glob.MARK)
+                ]
+            )
+        )
+        self.assertFalse(
+            all(
+                [
+                    glob.globmatch(
+                        x, '**/docs/**|!**/*.md', flags=self.flags | glob.SPLIT | glob.MARK
+                    ) for x in glob.glob('**/docs/**', flags=self.flags | glob.SPLIT | glob.MARK)
+                ]
+            )
+        )
+
     def test_glob_integrity_real(self):
         """`globmatch` must match what glob globs against the real file system."""
 
@@ -1025,6 +1069,49 @@ class TestGlobMatchSpecial(unittest.TestCase):
             )
         )
 
+    def test_glob_integrity_real_marked(self):
+        """`globmatch` must match what glob globs against the real file system and marked directories."""
+
+        # Number of slashes is inconsequential
+        # Glob really looks at what is in between. Multiple slashes are the same as one separator.
+        # UNC mounts are special cases and it matters there.
+        self.assertTrue(
+            all(
+                [
+                    glob.globmatch(
+                        x, '**/docs/**', flags=self.flags | glob.REALPATH | glob.MARK
+                    ) for x in glob.glob('**/docs/**', flags=self.flags | glob.MARK)
+                ]
+            )
+        )
+        self.assertTrue(
+            all(
+                [
+                    glob.globmatch(
+                        x, '**/docs/**|!**/*.md', flags=self.flags | glob.SPLIT | glob.REALPATH | glob.MARK
+                    ) for x in glob.glob('**/docs/**|!**/*.md', flags=self.flags | glob.SPLIT | glob.MARK)
+                ]
+            )
+        )
+        self.assertTrue(
+            all(
+                [
+                    glob.globmatch(
+                        x, '!**/*.md', flags=self.flags | glob.SPLIT | glob.REALPATH | glob.MARK
+                    ) for x in glob.glob('!**/*.md', flags=self.flags | glob.SPLIT | glob.MARK)
+                ]
+            )
+        )
+        self.assertFalse(
+            all(
+                [
+                    glob.globmatch(
+                        x, '**/docs/**|!**/*.md', flags=self.flags | glob.SPLIT | glob.REALPATH | glob.MARK
+                    ) for x in glob.glob('**/docs/**', flags=self.flags | glob.SPLIT | glob.MARK)
+                ]
+            )
+        )
+
     def test_glob_match_real_ignore_forcecase(self):
         """Ignore `FORCECASE` when using `globmatch` real."""