Repository: sqlfluff/sqlfluff

Issue:
Number of processes configurable in .sqlfluff
Being able to set the number of processes to run with in .sqlfluff might be useful to avoid having to pass it in the CLI every time.


Discussion:
One thought on this: The same `.sqlfluff` file will sometimes be used on different machines (e.g. various development machines, CI server). We should allow the setting to be somewhat "context sensitive" if desired. Proposal:
* Positive values indicate the number of processes to create
* Zero or negative values are interpreted as `number_of_cpus - specified_number`. Thus, a value of `0` means "use all processors" and `-1` means "use all processors except one".
Is there a standard way in python to detect the effective available cpus?
@alanmcruickshank: Yes. Use [`multiprocessing.cpu_count()`](https://docs.python.org/3/library/multiprocessing.html#multiprocessing.cpu_count).
I'm happy to pick this one up. This actually fits with a small reorg of how I think threads should be configured. I think it fits better if the thread argument is passed in to the `Linter` object on instantiation, rather than when calling `lint_paths`. @barrywhart - does that sit well with you? I realise that changes a little of the structure you originally envisaged.
Sounds good -- no concerns from me.

Patch:
diff --git a/src/sqlfluff/cli/commands.py b/src/sqlfluff/cli/commands.py
--- a/src/sqlfluff/cli/commands.py
+++ b/src/sqlfluff/cli/commands.py
@@ -486,8 +486,12 @@ def dump_file_payload(filename: Optional[str], payload: str):
     "-p",
     "--processes",
     type=int,
-    default=1,
-    help="The number of parallel processes to run.",
+    default=None,
+    help=(
+        "The number of parallel processes to run. Positive numbers work as "
+        "expected. Zero and negative numbers will work as number_of_cpus - "
+        "number. e.g  -1 means all cpus except one. 0 means all cpus."
+    ),
 )
 @click.option(
     "--disable_progress_bar",
@@ -497,7 +501,6 @@ def dump_file_payload(filename: Optional[str], payload: str):
 @click.argument("paths", nargs=-1, type=click.Path(allow_dash=True))
 def lint(
     paths: Tuple[str],
-    processes: int,
     format: str,
     write_output: Optional[str],
     annotation_level: str,
@@ -505,6 +508,7 @@ def lint(
     disregard_sqlfluffignores: bool,
     logger: Optional[logging.Logger] = None,
     bench: bool = False,
+    processes: Optional[int] = None,
     disable_progress_bar: Optional[bool] = False,
     extra_config_path: Optional[str] = None,
     ignore_local_config: bool = False,
@@ -675,8 +679,12 @@ def do_fixes(lnt, result, formatter=None, **kwargs):
     "-p",
     "--processes",
     type=int,
-    default=1,
-    help="The number of parallel processes to run.",
+    default=None,
+    help=(
+        "The number of parallel processes to run. Positive numbers work as "
+        "expected. Zero and negative numbers will work as number_of_cpus - "
+        "number. e.g  -1 means all cpus except one. 0 means all cpus."
+    ),
 )
 @click.option(
     "--disable_progress_bar",
@@ -705,10 +713,10 @@ def do_fixes(lnt, result, formatter=None, **kwargs):
 def fix(
     force: bool,
     paths: Tuple[str],
-    processes: int,
     bench: bool = False,
     fixed_suffix: str = "",
     logger: Optional[logging.Logger] = None,
+    processes: Optional[int] = None,
     disable_progress_bar: Optional[bool] = False,
     extra_config_path: Optional[str] = None,
     ignore_local_config: bool = False,
diff --git a/src/sqlfluff/cli/formatters.py b/src/sqlfluff/cli/formatters.py
--- a/src/sqlfluff/cli/formatters.py
+++ b/src/sqlfluff/cli/formatters.py
@@ -181,6 +181,14 @@ def dispatch_compilation_header(self, templater, message):
             f"=== [{self.colorize(templater, Color.lightgrey)}] {message}"
         )  # pragma: no cover
 
+    def dispatch_processing_header(self, processes: int) -> None:
+        """Dispatch the header displayed before linting."""
+        if self._verbosity > 0:
+            self._dispatch(  # pragma: no cover
+                f"{self.colorize('effective configured processes: ', Color.lightgrey)} "
+                f"{processes}"
+            )
+
     def dispatch_dialect_warning(self, dialect) -> None:
         """Dispatch a warning for dialects."""
         self._dispatch(self.format_dialect_warning(dialect))  # pragma: no cover
diff --git a/src/sqlfluff/core/linter/linter.py b/src/sqlfluff/core/linter/linter.py
--- a/src/sqlfluff/core/linter/linter.py
+++ b/src/sqlfluff/core/linter/linter.py
@@ -1066,7 +1066,7 @@ def lint_path(
         fix: bool = False,
         ignore_non_existent_files: bool = False,
         ignore_files: bool = True,
-        processes: int = 1,
+        processes: Optional[int] = None,
     ) -> LintedDir:
         """Lint a path."""
         linted_path = LintedDir(path)
@@ -1080,16 +1080,22 @@ def lint_path(
             )
         )
 
+        if processes is None:
+            processes = self.config.get("processes", default=1)
+
         # to avoid circular import
         from sqlfluff.core.linter.runner import get_runner
 
-        runner = get_runner(
+        runner, effective_processes = get_runner(
             self,
             self.config,
             processes=processes,
             allow_process_parallelism=self.allow_process_parallelism,
         )
 
+        if self.formatter and effective_processes != 1:
+            self.formatter.dispatch_processing_header(effective_processes)
+
         # Show files progress bar only when there is more than one.
         files_count = len(fnames)
         progress_bar_files = tqdm(
@@ -1124,7 +1130,7 @@ def lint_paths(
         fix: bool = False,
         ignore_non_existent_files: bool = False,
         ignore_files: bool = True,
-        processes: int = 1,
+        processes: Optional[int] = None,
     ) -> LintingResult:
         """Lint an iterable of paths."""
         paths_count = len(paths)
diff --git a/src/sqlfluff/core/linter/runner.py b/src/sqlfluff/core/linter/runner.py
--- a/src/sqlfluff/core/linter/runner.py
+++ b/src/sqlfluff/core/linter/runner.py
@@ -10,6 +10,7 @@
 import bdb
 import functools
 import logging
+import multiprocessing
 import multiprocessing.dummy
 import signal
 import sys
@@ -227,15 +228,29 @@ def get_runner(
     config: FluffConfig,
     processes: int,
     allow_process_parallelism: bool = True,
-) -> BaseRunner:
-    """Generate a runner instance based on parallel and system configuration."""
+) -> Tuple[BaseRunner, int]:
+    """Generate a runner instance based on parallel and system configuration.
+
+    The processes argument can be positive or negative.
+    - If positive, the integer is interpreted as the number of processes.
+    - If negative or zero, the integer is interpreted as number_of_cpus - processes.
+
+    e.g.
+    -1 = all cpus but one.
+    0 = all cpus
+    1 = 1 cpu
+
+    """
+    if processes <= 0:
+        processes = max(multiprocessing.cpu_count() + processes, 1)
+
     if processes > 1:
         # Process parallelism isn't really supported during testing
         # so this flag allows us to fall back to a threaded runner
         # in those cases.
         if allow_process_parallelism:
-            return MultiProcessRunner(linter, config, processes=processes)
+            return MultiProcessRunner(linter, config, processes=processes), processes
         else:
-            return MultiThreadRunner(linter, config, processes=processes)
+            return MultiThreadRunner(linter, config, processes=processes), processes
     else:
-        return SequentialRunner(linter, config)
+        return SequentialRunner(linter, config), processes


Test patch:
diff --git a/test/core/linter_test.py b/test/core/linter_test.py
--- a/test/core/linter_test.py
+++ b/test/core/linter_test.py
@@ -14,6 +14,7 @@
 from sqlfluff.cli.formatters import OutputStreamFormatter
 from sqlfluff.cli.outputstream import make_output_stream
 from sqlfluff.core.linter import LintingResult, NoQaDirective
+from sqlfluff.core.linter.runner import get_runner
 import sqlfluff.core.linter as linter
 from sqlfluff.core.parser import GreedyUntil, Ref
 from sqlfluff.core.templaters import TemplatedFile
@@ -289,6 +290,36 @@ def test_lint_path_parallel_wrapper_exception(patched_lint):
             result.reraise()
 
 
+@pytest.mark.parametrize(
+    "mock_cpu,in_processes,exp_processes",
+    [
+        # Make the mocked cpu count a really high value which is
+        # unlikely to collide with the real value. We can then
+        # test all the different combos.
+        (512, 1, 1),
+        (512, 0, 512),
+        (512, -12, 500),
+        (512, 5, 5),
+        # Check that we can't go lower than 1 in a 1 cpu case
+        (1, -1, 1),
+    ],
+)
+@patch("multiprocessing.cpu_count")
+def test__linter__get_runner_processes(
+    patched_cpu_count, mock_cpu, in_processes, exp_processes
+):
+    """Test that get_runner handles processes correctly."""
+    # Make the mocked cpu count a really high value which is
+    # unlikely to collide with the real value.
+    patched_cpu_count.return_value = mock_cpu
+    _, return_processes = get_runner(
+        linter=Linter(),
+        config=FluffConfig(overrides={"dialect": "ansi"}),
+        processes=in_processes,
+    )
+    assert return_processes == exp_processes
+
+
 @patch("sqlfluff.core.linter.runner.linter_logger")
 @patch("sqlfluff.core.linter.Linter.lint_rendered")
 def test__linter__linting_unexpected_error_handled_gracefully(