Repository: pantsbuild/pants

Issue:
[pantsd] Integration tests under chroot are broken with pantsd
## Problem

In CI, we run a cron job every day to check that master runs. In that cron job, we run the same integration test shards with and without pantsd.
Since we started introducing integration tests under chroot, those shards have been broken.
The main problem seems to be that `Watchman` behaves baldly under chroot ([relevant thread](https://pantsbuild.slack.com/archives/C15HUUSL9/p1574344335019500)).
As far as we know, these are the concrete problems:

- When using with chroot, the watchman socket path ends up being too long.
- When using with chroot, the folder we chroot to seems to dissapear before we properly close out pantsd, crashing watchman.

## Repro
Run integration tests with the flag `--test-pytest-chroot` and the environment variable `USE_PANTSD_FOR_INTEGRATION_TESTS="true"`

plumb `--pants-ignore` to watchman startup.
currently this is hardcoded with two outstanding TODOs:

```
service/fs_event_service.py
82:          # TODO(kwlzn): Make exclusions here optionable.

subsystem/pants_daemon_launcher.py
99:    # TODO(kwlzn): Thread filesystem path ignores here to Watchman's subscription registration.
```



Discussion:

I have a functional workup on this, but I'm going to change the approach to use `pathspec` - after knocking out some other more important tasks.

Relates to #4999

Resolving patch:
diff --git a/src/python/pants/engine/internals/native.py b/src/python/pants/engine/internals/native.py
--- a/src/python/pants/engine/internals/native.py
+++ b/src/python/pants/engine/internals/native.py
@@ -1003,7 +1003,6 @@ def ti(type_obj):
             execution_options.process_execution_use_local_cache,
             self.context.utf8_dict(execution_options.remote_execution_headers),
             execution_options.process_execution_local_enable_nailgun,
-            execution_options.experimental_fs_watcher,
         )
         if scheduler_result.is_throw:
             value = self.context.from_value(scheduler_result.throw_handle)
diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py
--- a/src/python/pants/option/global_options.py
+++ b/src/python/pants/option/global_options.py
@@ -102,7 +102,6 @@ class ExecutionOptions:
     remote_execution_extra_platform_properties: Any
     remote_execution_headers: Any
     process_execution_local_enable_nailgun: bool
-    experimental_fs_watcher: bool
 
     @classmethod
     def from_bootstrap_options(cls, bootstrap_options):
@@ -128,7 +127,6 @@ def from_bootstrap_options(cls, bootstrap_options):
             remote_execution_extra_platform_properties=bootstrap_options.remote_execution_extra_platform_properties,
             remote_execution_headers=bootstrap_options.remote_execution_headers,
             process_execution_local_enable_nailgun=bootstrap_options.process_execution_local_enable_nailgun,
-            experimental_fs_watcher=bootstrap_options.experimental_fs_watcher,
         )
 
 
@@ -154,7 +152,6 @@ def from_bootstrap_options(cls, bootstrap_options):
     remote_execution_extra_platform_properties=[],
     remote_execution_headers={},
     process_execution_local_enable_nailgun=False,
-    experimental_fs_watcher=True,
 )
 
 
@@ -665,9 +662,11 @@ def register_bootstrap_options(cls, register):
             "--watchman-enable",
             type=bool,
             advanced=True,
-            default=True,
+            default=False,
+            removal_version="1.30.0.dev0",
+            removal_hint="The native watcher is now sufficient to monitor for filesystem changes.",
             help="Use the watchman daemon filesystem event watcher to watch for changes "
-            "in the buildroot. Disable this to rely solely on the experimental pants engine filesystem watcher.",
+            "in the buildroot in addition to the built in watcher.",
         )
         register(
             "--watchman-version", advanced=True, default="4.9.0-pants1", help="Watchman version."
@@ -903,6 +902,8 @@ def register_bootstrap_options(cls, register):
             type=bool,
             default=True,
             advanced=True,
+            removal_version="1.30.0.dev0",
+            removal_hint="Enabled by default: flag is disabled.",
             help="Whether to use the engine filesystem watcher which registers the workspace"
             " for kernel file change events",
         )
@@ -1063,6 +1064,8 @@ def validate_instance(cls, opts):
         Raises pants.option.errors.OptionsError on validation failure.
         """
         if opts.get("loop") and not opts.enable_pantsd:
+            # TODO: This remains the case today because there are two spots that
+            # call `run_goal_rules`: fixing in a followup.
             raise OptionsError(
                 "The `--loop` option requires `--enable-pantsd`, in order to watch files."
             )


Test patch:
diff --git a/src/rust/engine/watch/src/tests.rs b/src/rust/engine/watch/src/tests.rs
--- a/src/rust/engine/watch/src/tests.rs
+++ b/src/rust/engine/watch/src/tests.rs
@@ -33,7 +33,7 @@ async fn setup_watch(
   file_path: PathBuf,
 ) -> Arc<InvalidationWatcher> {
   let executor = Executor::new(Handle::current());
-  let watcher = InvalidationWatcher::new(executor, build_root, ignorer, /*enabled*/ true)
+  let watcher = InvalidationWatcher::new(executor, build_root, ignorer)
     .expect("Couldn't create InvalidationWatcher");
   watcher.watch(file_path).await.unwrap();
   watcher
diff --git a/tests/python/pants_test/init/test_options_initializer.py b/tests/python/pants_test/init/test_options_initializer.py
--- a/tests/python/pants_test/init/test_options_initializer.py
+++ b/tests/python/pants_test/init/test_options_initializer.py
@@ -22,16 +22,9 @@ def test_invalid_version(self):
     def test_global_options_validation(self):
         # Specify an invalid combination of options.
         ob = OptionsBootstrapper.create(
-            args=[
-                "--backend-packages=[]",
-                "--backend-packages2=[]",
-                "--v2",
-                "--no-v1",
-                "--loop",
-                "--no-enable-pantsd",
-            ]
+            args=["--backend-packages=[]", "--backend-packages2=[]", "--remote-execution",]
         )
         build_config = BuildConfigInitializer.get(ob)
         with self.assertRaises(OptionsError) as exc:
             OptionsInitializer.create(ob, build_config)
-        self.assertIn("The `--loop` option requires `--enable-pantsd`", str(exc.exception))
+        self.assertIn("The `--remote-execution` option requires", str(exc.exception))
diff --git a/tests/python/pants_test/pantsd/test_watchman_launcher.py b/tests/python/pants_test/pantsd/test_watchman_launcher.py
--- a/tests/python/pants_test/pantsd/test_watchman_launcher.py
+++ b/tests/python/pants_test/pantsd/test_watchman_launcher.py
@@ -10,7 +10,8 @@
 
 class TestWatchmanLauncher(TestBase):
     def watchman_launcher(self, cli_options=()):
-        bootstrap_options = self.get_bootstrap_options(cli_options)
+        options = ("--watchman-enable", *cli_options)
+        bootstrap_options = self.get_bootstrap_options(options)
         return WatchmanLauncher.create(bootstrap_options)
 
     def create_mock_watchman(self, is_alive):