Repository: pantsbuild/pants

Issue:
Port owners-discovery PathGlob matching to rust
Discovering which targets own a particular file spec which may not exist on disk (as discussed in https://github.com/pantsbuild/pants/pull/6790/files#r234578507) requires matching a path against the string form of a rust `PathGlob` (called a `filespec` on the python side).

Porting this code to rust would unify some [complex python code](https://github.com/pantsbuild/pants/blob/550e4cd0d03297a3b92e1d7030b28818909f0822/src/python/pants/source/filespec.py) (which dynamically compiles regexes) with our existing `PathGlob` matching code on the rust side.

This is important, but non-trivial. The `PathGlob` matching code [expands globs while traversing the filesystem](https://github.com/pantsbuild/pants/blob/550e4cd0d03297a3b92e1d7030b28818909f0822/src/rust/engine/fs/src/glob_matching.rs). Cleanly decoupling this would be lovely, but potentially challenging.


Discussion:
...a simple, strawman implementation of this would be to create an in-memory implementation of `VFS` that just contained the concrete list of the files you would like to match. You'd then run `expand` inside that VFS. This is overkill (because if you have the full file listing ahead of time, there is no point in all of the laziness that `VFS::expand` uses), but as a first cut, it would work, and accomplish the goal.

There isn't a distinct need for speed for this code, because it generally operates on a small fraction of the repo: it would be matching some sources against the PathGlobs of all targets defined between those files and the root of the repo.
As a first cut, I think just exposing `PathGlob::matches` which forwards to `PathGlob.wildcard.matches` would be an improvement on today...
Mm, yea... possibly. Due to how PathGlobs are expanded, we currently never match a [glob::Pattern](https://docs.rs/glob/0.2.11/glob/struct.Pattern.html) against a complete file path (only individual components) so I'm not sure whether Pattern's syntax is the exact syntax we support.

But moving toward a world where we have one codepath that interprets PathGlobs to lazily traverse the filesystem to emit a stream of candidates, and then a separate codepath that re-matches on the complete Path relative to the buildroot... might be a good decoupling?

Resolving patch:
diff --git a/src/python/pants/engine/native.py b/src/python/pants/engine/native.py
--- a/src/python/pants/engine/native.py
+++ b/src/python/pants/engine/native.py
@@ -674,6 +674,12 @@ def decompress_tarball(self, tarfile_path, dest_dir):
     result = self.lib.decompress_tarball(tarfile_path, dest_dir)
     return self.context.raise_or_return(result)
 
+  def match_path_globs(self, path_globs, paths):
+    path_globs = self.context.to_value(path_globs)
+    paths_buf = self.context.utf8_buf_buf(tuple(paths))
+    result = self.lib.match_path_globs(path_globs, paths_buf)
+    return self.context.raise_or_return(result)
+
   def new_tasks(self):
     return self.gc(self.lib.tasks_create(), self.lib.tasks_destroy)
 
diff --git a/src/python/pants/source/filespec.py b/src/python/pants/source/filespec.py
--- a/src/python/pants/source/filespec.py
+++ b/src/python/pants/source/filespec.py
@@ -4,60 +4,13 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-import re
-
-
-def glob_to_regex(pattern):
-  """Given a glob pattern, return an equivalent regex expression.
-
-  TODO: Replace with implementation in `fs.rs`. See https://github.com/pantsbuild/pants/issues/6795.
-
-  :param string glob: The glob pattern. "**" matches 0 or more dirs recursively.
-                      "*" only matches patterns in a single dir.
-  :returns: A regex string that matches same paths as the input glob does.
-  """
-  out = ['^']
-  components = pattern.strip('/').replace('.', '[.]').replace('$','[$]').split('/')
-  doublestar = False
-  for component in components:
-    if len(out) == 1:
-      if pattern.startswith('/'):
-        out.append('/')
-    else:
-      if not doublestar:
-        out.append('/')
-
-    if '**' in component:
-      if component != '**':
-        raise ValueError('Invalid usage of "**", use "*" instead.')
-
-      if not doublestar:
-        out.append('(([^/]+/)*)')
-        doublestar = True
-    else:
-      out.append(component.replace('*', '[^/]*'))
-      doublestar = False
-
-  if doublestar:
-    out.append('[^/]*')
-
-  out.append('$')
-
-  return ''.join(out)
+from pants.engine.fs import PathGlobs
+from pants.engine.native import Native
 
 
 def globs_matches(paths, patterns, exclude_patterns):
-  def excluded(path):
-    if excluded.regexes is None:
-      excluded.regexes = [re.compile(glob_to_regex(ex)) for ex in exclude_patterns]
-    return any(ex.match(path) for ex in excluded.regexes)
-  excluded.regexes = None
-  for pattern in patterns:
-    regex = re.compile(glob_to_regex(pattern))
-    for path in paths:
-      if regex.match(path) and not excluded(path):
-        return True
-  return False
+  path_globs = PathGlobs(include=patterns, exclude=exclude_patterns)
+  return Native().match_path_globs(path_globs, paths)
 
 
 def matches_filespec(path, spec):
@@ -65,8 +18,6 @@ def matches_filespec(path, spec):
 
 
 def any_matches_filespec(paths, spec):
-  if not paths or not spec:
-    return False
   exclude_patterns = []
   for exclude_spec in spec.get('exclude', []):
     exclude_patterns.extend(exclude_spec.get('globs', []))


Test patch:
diff --git a/tests/python/pants_test/source/test_filespec.py b/tests/python/pants_test/source/test_filespec.py
--- a/tests/python/pants_test/source/test_filespec.py
+++ b/tests/python/pants_test/source/test_filespec.py
@@ -4,111 +4,96 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-import re
 import unittest
 
-from pants.source.filespec import glob_to_regex
+from pants.source.filespec import matches_filespec
 
 
-class GlobToRegexTest(unittest.TestCase):
+class FilespecTest(unittest.TestCase):
   def assert_rule_match(self, glob, expected_matches, negate=False):
     if negate:
-      asserter, match_state = self.assertIsNone, 'erroneously matches'
+      asserter, match_state = self.assertFalse, 'erroneously matches'
     else:
-      asserter, match_state = self.assertIsNotNone, "doesn't match"
+      asserter, match_state = self.assertTrue, "doesn't match"
 
-    regex = glob_to_regex(glob)
     for expected in expected_matches:
-      asserter(re.match(regex, expected), 'glob_to_regex(`{}`) -> `{}` {} path `{}`'
-                                          .format(glob, regex, match_state, expected))
+      asserter(
+          matches_filespec(expected, {'globs': [glob]}),
+          '{} {} path `{}`'.format(glob, match_state, expected),
+      )
 
-  def test_glob_to_regex_single_star_0(self):
+  def test_matches_single_star_0(self):
     self.assert_rule_match('a/b/*/f.py', ('a/b/c/f.py', 'a/b/q/f.py'))
 
-  def test_glob_to_regex_single_star_0_neg(self):
+  def test_matches_single_star_0_neg(self):
     self.assert_rule_match('a/b/*/f.py', ('a/b/c/d/f.py','a/b/f.py'), negate=True)
 
-  def test_glob_to_regex_single_star_1(self):
+  def test_matches_single_star_1(self):
     self.assert_rule_match('foo/bar/*', ('foo/bar/baz', 'foo/bar/bar'))
 
-  def test_glob_to_regex_single_star_2(self):
+  def test_matches_single_star_2(self):
     self.assert_rule_match('*/bar/b*', ('foo/bar/baz', 'foo/bar/bar'))
 
-  def test_glob_to_regex_single_star_2_neg(self):
+  def test_matches_single_star_2_neg(self):
     self.assert_rule_match('*/bar/b*', ('foo/koo/bar/baz', 'foo/bar/bar/zoo'), negate=True)
 
-  def test_glob_to_regex_single_star_3(self):
-    self.assert_rule_match('/*/[be]*/b*', ('/foo/bar/baz', '/foo/bar/bar'))
+  def test_matches_single_star_3(self):
+    self.assert_rule_match('*/[be]*/b*', ('foo/bar/baz', 'foo/bar/bar'))
 
-  def test_glob_to_regex_single_star_4(self):
-    self.assert_rule_match('/foo*/bar', ('/foofighters/bar', '/foofighters.venv/bar'))
+  def test_matches_single_star_4(self):
+    self.assert_rule_match('foo*/bar', ('foofighters/bar', 'foofighters.venv/bar'))
 
-  def test_glob_to_regex_single_star_4_neg(self):
-    self.assert_rule_match('/foo*/bar', ('/foofighters/baz/bar',), negate=True)
+  def test_matches_single_star_4_neg(self):
+    self.assert_rule_match('foo*/bar', ('foofighters/baz/bar',), negate=True)
 
-  def test_glob_to_regex_double_star_0(self):
+  def test_matches_double_star_0(self):
     self.assert_rule_match('**', ('a/b/c', 'a'))
 
-  def test_glob_to_regex_double_star_1(self):
+  def test_matches_double_star_1(self):
     self.assert_rule_match('a/**/f', ('a/f', 'a/b/c/d/e/f'))
 
-  def test_glob_to_regex_double_star_2(self):
+  def test_matches_double_star_2(self):
     self.assert_rule_match('a/b/**', ('a/b/c', 'a/b/c/d/e/f'))
 
-  def test_glob_to_regex_double_star_2_neg(self):
+  def test_matches_double_star_2_neg(self):
     self.assert_rule_match('a/b/**', ('a/b'), negate=True)
 
-  def test_glob_to_regex_leading_slash_0(self):
-    self.assert_rule_match('/a/*', ('/a/a', '/a/b.py'))
+  def test_matches_dots(self):
+    self.assert_rule_match('.*', ('.pants.d', '.pids'))
 
-  def test_glob_to_regex_leading_slash_0_neg(self):
-    self.assert_rule_match('/a/*', ('a/a', 'a/b.py'), negate=True)
+  def test_matches_dots_relative(self):
+    self.assert_rule_match('./*.py', ('f.py', 'g.py'))
 
-  def test_glob_to_regex_leading_slash_1(self):
-    self.assert_rule_match('/*', ('/a', '/a.py'))
-
-  def test_glob_to_regex_leading_slash_1_neg(self):
-    self.assert_rule_match('/*', ('a', 'a.py'), negate=True)
-
-  def test_glob_to_regex_leading_slash_2(self):
-    self.assert_rule_match('/**', ('/a', '/a/b/c/d/e/f'))
-
-  def test_glob_to_regex_leading_slash_2_neg(self):
-    self.assert_rule_match('/**', ('a', 'a/b/c/d/e/f'), negate=True)
-
-  def test_glob_to_regex_dots(self):
-    self.assert_rule_match('.*', ('.pants.d', '.', '..', '.pids'))
-
-  def test_glob_to_regex_dots_neg(self):
+  def test_matches_dots_neg(self):
     self.assert_rule_match(
       '.*',
       ('a', 'a/non/dot/dir/file.py', 'dist', 'all/nested/.dot', '.some/hidden/nested/dir/file.py'),
       negate=True
     )
 
-  def test_glob_to_regex_dirs(self):
+  def test_matches_dirs(self):
     self.assert_rule_match('dist/', ('dist',))
 
-  def test_glob_to_regex_dirs_neg(self):
+  def test_matches_dirs_neg(self):
     self.assert_rule_match('dist/', ('not_dist', 'cdist', 'dist.py', 'dist/dist'), negate=True)
 
-  def test_glob_to_regex_dirs_dots(self):
+  def test_matches_dirs_dots(self):
     self.assert_rule_match(
       'build-support/*.venv/',
       ('build-support/*.venv',
        'build-support/rbt.venv')
     )
 
-  def test_glob_to_regex_dirs_dots_neg(self):
+  def test_matches_dirs_dots_neg(self):
     self.assert_rule_match('build-support/*.venv/',
                            ('build-support/rbt.venv.but_actually_a_file',),
                            negate=True)
 
-  def test_glob_to_regex_literals(self):
+  def test_matches_literals(self):
     self.assert_rule_match('a', ('a',))
 
-  def test_glob_to_regex_literal_dir(self):
+  def test_matches_literal_dir(self):
     self.assert_rule_match('a/b/c', ('a/b/c',))
 
-  def test_glob_to_regex_literal_file(self):
+  def test_matches_literal_file(self):
     self.assert_rule_match('a/b/c.py', ('a/b/c.py',))