Repository: pantsbuild/pants

Issue:
Invalid isort configuration is silently ignored
**Describe the bug**

When an isort config file (`.isort.cfg` or `pyproject.toml`) is invalid, `isort` just emits a warning, and the whole invocation succeeds (exit code = 0). The invocation succeeding means `pants fmt ::` won't surface the warning, and thus people may be unintentionally running with a default isort configuration if they made a mistake while editing the configuration.

NB. fixing this to be an error instead may be a breaking change, since `pants fmt ::` invocations with invalid config were previously succeeding and would start failing.

Reproducer, configured to generate an invalid `.isort.cfg` by default, but can be switched to `pyproject.toml`:


```shell
cat > pants.toml <<EOF
[GLOBAL]
pants_version = "2.15.0"

backend_packages = [
  "pants.backend.python",
  "pants.backend.python.lint.isort",
]

[anonymous-telemetry]
enabled = false
EOF

if true; then # switch to false to generate pyproject.toml
  cat > .isort.cfg <<EOF
[settings]
force_single_line = true
# invalid setting:
no_sections = this should be a bool, but isnt
EOF
else
  cat > pyproject.toml <<EOF
[tool.isort]
force_single_line = true
# invalid setting:
no_sections = "this should be a bool, but isnt"
EOF
fi

cat > example.py <<EOF
from foo import bar, baz
EOF

echo "BUG: this does _NOT_ fix example.py"
pants fmt ::

cat example.py

echo "Running outside of pants:"
pants export --resolve=isort
dist/export/python/virtualenvs/isort/*/bin/isort .
```

Running `bash run.sh` gives output:

```
BUG: this does _NOT_ fix example.py
09:12:01.84 [INFO] Initializing scheduler...
09:12:02.27 [INFO] Scheduler initialized.
from foo import bar, baz
```
```
Running outside of pants:
Wrote mutable virtualenv for isort (using Python 3.7.13) to dist/export/python/virtualenvs/isort
/private/var/folders/sv/vd266m4d4lvctgs2wpnhjs9w0000gn/T/tmp.0BEDipYZ/dist/export/python/virtualenvs/isort/3.7.13/lib/python3.7/site-packages/isort/settings.py:771: UserWarning: Failed to pull configuration information from /private/var/folders/sv/vd266m4d4lvctgs2wpnhjs9w0000gn/T/tmp.0BEDipYZ/.isort.cfg
  warn(f"Failed to pull configuration information from {potential_config_file}")
Skipped 2 files
```

In particular, note:
- the `example.py` isn't changed: it _should_ become `from foo import bar` `from foo import baz` on separate lines due to `force_single_line = true`
- running isort directly gives a `Failed to pull configuration information from ...` _warning_

**Pants version**
2.15.0

**OS**
macOS

**Additional info**
N/A


Discussion:
In general we should be surfacing warnings from any process, I think. Certainly from any linter.

Resolving patch:
diff --git a/src/python/pants/backend/python/lint/isort/rules.py b/src/python/pants/backend/python/lint/isort/rules.py
--- a/src/python/pants/backend/python/lint/isort/rules.py
+++ b/src/python/pants/backend/python/lint/isort/rules.py
@@ -14,9 +14,10 @@
 from pants.core.util_rules.config_files import ConfigFiles, ConfigFilesRequest
 from pants.core.util_rules.partitions import PartitionerType
 from pants.engine.fs import Digest, MergeDigests
-from pants.engine.process import ProcessResult
+from pants.engine.process import ProcessExecutionFailure, ProcessResult
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
 from pants.engine.target import FieldSet, Target
+from pants.option.global_options import KeepSandboxes
 from pants.util.logging import LogLevel
 from pants.util.strutil import pluralize
 
@@ -62,7 +63,9 @@ def generate_argv(
 
 
 @rule(desc="Format with isort", level=LogLevel.DEBUG)
-async def isort_fmt(request: IsortRequest.Batch, isort: Isort) -> FmtResult:
+async def isort_fmt(
+    request: IsortRequest.Batch, isort: Isort, keep_sandboxes: KeepSandboxes
+) -> FmtResult:
     isort_pex_get = Get(VenvPex, PexRequest, isort.to_pex_request())
     config_files_get = Get(
         ConfigFiles, ConfigFilesRequest, isort.config_request(request.snapshot.dirs)
@@ -80,6 +83,7 @@ async def isort_fmt(request: IsortRequest.Batch, isort: Isort) -> FmtResult:
         Digest, MergeDigests((request.snapshot.digest, config_files.snapshot.digest))
     )
 
+    description = f"Run isort on {pluralize(len(request.files), 'file')}."
     result = await Get(
         ProcessResult,
         VenvPexProcess(
@@ -87,10 +91,20 @@ async def isort_fmt(request: IsortRequest.Batch, isort: Isort) -> FmtResult:
             argv=generate_argv(request.files, isort, is_isort5=is_isort5),
             input_digest=input_digest,
             output_files=request.files,
-            description=f"Run isort on {pluralize(len(request.files), 'file')}.",
+            description=description,
             level=LogLevel.DEBUG,
         ),
     )
+
+    if b"Failed to pull configuration information" in result.stderr:
+        raise ProcessExecutionFailure(
+            -1,
+            result.stdout,
+            result.stderr,
+            description,
+            keep_sandboxes=keep_sandboxes,
+        )
+
     return await FmtResult.create(request, result, strip_chroot_path=True)
 
 


Test patch:
diff --git a/src/python/pants/backend/python/lint/isort/rules_integration_test.py b/src/python/pants/backend/python/lint/isort/rules_integration_test.py
--- a/src/python/pants/backend/python/lint/isort/rules_integration_test.py
+++ b/src/python/pants/backend/python/lint/isort/rules_integration_test.py
@@ -3,6 +3,9 @@
 
 from __future__ import annotations
 
+import re
+from textwrap import dedent
+
 import pytest
 
 from pants.backend.python import target_types_rules
@@ -15,6 +18,7 @@
 from pants.core.util_rules import config_files, source_files
 from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
 from pants.engine.addresses import Address
+from pants.engine.internals.scheduler import ExecutionError
 from pants.engine.target import Target
 from pants.testutil.python_interpreter_selection import all_major_minor_python_versions
 from pants.testutil.rule_runner import QueryRule, RuleRunner
@@ -137,6 +141,32 @@ def test_config_file(rule_runner: RuleRunner, path: str, extra_args: list[str])
     assert fmt_result.did_change is True
 
 
+def test_invalid_config_file(rule_runner: RuleRunner) -> None:
+    """Reference https://github.com/pantsbuild/pants/issues/18618."""
+
+    rule_runner.write_files(
+        {
+            ".isort.cfg": dedent(
+                """\
+        [settings]
+        force_single_line = true
+        # invalid setting:
+        no_sections = this should be a bool, but isnt
+        """
+            ),
+            "example.py": "from foo import bar, baz",
+            "BUILD": "python_sources()",
+        }
+    )
+    tgt = rule_runner.get_target((Address("", relative_file_path="example.py")))
+    with pytest.raises(ExecutionError) as isort_error:
+        run_isort(rule_runner, [tgt])
+    assert any(
+        re.search(r"Failed to pull configuration information from .*\.isort\.cfg", arg)
+        for arg in isort_error.value.args
+    )
+
+
 def test_passthrough_args(rule_runner: RuleRunner) -> None:
     rule_runner.write_files({"f.py": NEEDS_CONFIG_FILE, "BUILD": "python_sources(name='t')"})
     tgt = rule_runner.get_target(Address("", target_name="t", relative_file_path="f.py"))