Repository: sqlfluff/sqlfluff

Issue:
Multiple processes not used when list of explicit filenames is passed
### Search before asking

- [X] I searched the [issues](https://github.com/sqlfluff/sqlfluff/issues) and found no similar issues.


### What Happened

When providing a long list of file names to `sqlfluff lint -p -1`, only a single CPU is used. This seems to stem from the fact that https://github.com/sqlfluff/sqlfluff/blob/a006378af8b670f9235653694dbcddd4c62d1ab9/src/sqlfluff/core/linter/linter.py#L1190 is iterating over the list of files. For each listed path there, it would run the found files in parallel. As we are inputting whole filenames here, a path equals a single file and thus `sqlfluff` would only process one file at a time.

The context here is the execution of `sqlfluff lint` inside a `pre-commit` hook.

### Expected Behaviour

All CPU cores are used as `-p -1` is passed on the commandline.

### Observed Behaviour

Only a single CPU core is used.

### How to reproduce

Run `sqlfluff lint -p -1` with a long list of files.

### Dialect

Affects all. 

### Version

1.4.2

### Configuration

None.

### Are you willing to work on and submit a PR to address the issue?

- [ ] Yes I am willing to submit a PR!

### Code of Conduct

- [X] I agree to follow this project's [Code of Conduct](https://github.com/sqlfluff/sqlfluff/blob/main/CODE_OF_CONDUCT.md)



Discussion:
I have been wondering for some time why sqlfluff never manages to use 100% of CPU. Running it on my Code base takes about 90 minutes. Though never more than 30% of cpu is used… maybe this sis the reason…
Yeah - this looks like an accurate diagnosis. Most of the testing for the multiprocessing feature was done on large projects of multiple files, but _where a single path was passed_ e.g. `sqlfluff lint .`.

This seems like a very sensible improvement for people using the commit hook.

@barrywhart - you did a lot of the original multiprocessing work. Reckon you could take this one on?
I'll take a look, sure!

Patch:
diff --git a/src/sqlfluff/core/linter/linter.py b/src/sqlfluff/core/linter/linter.py
--- a/src/sqlfluff/core/linter/linter.py
+++ b/src/sqlfluff/core/linter/linter.py
@@ -1103,17 +1103,39 @@ def lint_path(
         processes: Optional[int] = None,
     ) -> LintedDir:
         """Lint a path."""
-        linted_path = LintedDir(path)
-        if self.formatter:
-            self.formatter.dispatch_path(path)
-        fnames = list(
-            self.paths_from_path(
+        return self.lint_paths(
+            (path,), fix, ignore_non_existent_files, ignore_files, processes
+        ).paths[0]
+
+    def lint_paths(
+        self,
+        paths: Tuple[str, ...],
+        fix: bool = False,
+        ignore_non_existent_files: bool = False,
+        ignore_files: bool = True,
+        processes: Optional[int] = None,
+    ) -> LintingResult:
+        """Lint an iterable of paths."""
+        # If no paths specified - assume local
+        if not paths:  # pragma: no cover
+            paths = (os.getcwd(),)
+        # Set up the result to hold what we get back
+        result = LintingResult()
+
+        expanded_paths: List[str] = []
+        expanded_path_to_linted_dir = {}
+        for path in paths:
+            linted_dir = LintedDir(path)
+            result.add(linted_dir)
+            for fname in self.paths_from_path(
                 path,
                 ignore_non_existent_files=ignore_non_existent_files,
                 ignore_files=ignore_files,
-            )
-        )
+            ):
+                expanded_paths.append(fname)
+                expanded_path_to_linted_dir[fname] = linted_dir
 
+        files_count = len(expanded_paths)
         if processes is None:
             processes = self.config.get("processes", default=1)
 
@@ -1131,72 +1153,29 @@ def lint_path(
             self.formatter.dispatch_processing_header(effective_processes)
 
         # Show files progress bar only when there is more than one.
-        files_count = len(fnames)
+        first_path = expanded_paths[0] if expanded_paths else ""
         progress_bar_files = tqdm(
             total=files_count,
-            desc=f"file {os.path.basename(fnames[0] if fnames else '')}",
+            desc=f"file {first_path}",
             leave=False,
             disable=files_count <= 1 or progress_bar_configuration.disable_progress_bar,
         )
 
-        for i, linted_file in enumerate(runner.run(fnames, fix), start=1):
-            linted_path.add(linted_file)
+        for i, linted_file in enumerate(runner.run(expanded_paths, fix), start=1):
+            linted_dir = expanded_path_to_linted_dir[linted_file.path]
+            linted_dir.add(linted_file)
             # If any fatal errors, then stop iteration.
             if any(v.fatal for v in linted_file.violations):  # pragma: no cover
                 linter_logger.error("Fatal linting error. Halting further linting.")
                 break
 
             # Progress bar for files is rendered only when there is more than one file.
-            # Additionally as it's updated after each loop, we need to get file name
+            # Additionally, as it's updated after each loop, we need to get file name
             # from the next loop. This is why `enumerate` starts with `1` and there
             # is `i < len` to not exceed files list length.
             progress_bar_files.update(n=1)
-            if i < len(fnames):
-                progress_bar_files.set_description(
-                    f"file {os.path.basename(fnames[i])}"
-                )
-
-        return linted_path
-
-    def lint_paths(
-        self,
-        paths: Tuple[str, ...],
-        fix: bool = False,
-        ignore_non_existent_files: bool = False,
-        ignore_files: bool = True,
-        processes: Optional[int] = None,
-    ) -> LintingResult:
-        """Lint an iterable of paths."""
-        paths_count = len(paths)
-
-        # If no paths specified - assume local
-        if not paths_count:  # pragma: no cover
-            paths = (os.getcwd(),)
-        # Set up the result to hold what we get back
-        result = LintingResult()
-
-        progress_bar_paths = tqdm(
-            total=paths_count,
-            desc="path",
-            leave=False,
-            disable=paths_count <= 1 or progress_bar_configuration.disable_progress_bar,
-        )
-        for path in paths:
-            progress_bar_paths.set_description(f"path {path}")
-
-            # Iterate through files recursively in the specified directory (if it's a
-            # directory) or read the file directly if it's not
-            result.add(
-                self.lint_path(
-                    path,
-                    fix=fix,
-                    ignore_non_existent_files=ignore_non_existent_files,
-                    ignore_files=ignore_files,
-                    processes=processes,
-                )
-            )
-
-            progress_bar_paths.update(1)
+            if i < len(expanded_paths):
+                progress_bar_files.set_description(f"file {expanded_paths[i]}")
 
         result.stop_timer()
         return result


Test patch:
diff --git a/test/cli/commands_test.py b/test/cli/commands_test.py
--- a/test/cli/commands_test.py
+++ b/test/cli/commands_test.py
@@ -1272,15 +1272,16 @@ def test__cli__command_lint_nocolor(isatty, should_strip_ansi, capsys, tmpdir):
 )
 @pytest.mark.parametrize("write_file", [None, "outfile"])
 def test__cli__command_lint_serialize_multiple_files(serialize, write_file, tmp_path):
-    """Test the output output formats for multiple files.
+    """Test the output formats for multiple files.
 
     This tests runs both stdout checking and file checking.
     """
-    fpath = "test/fixtures/linter/indentation_errors.sql"
+    fpath1 = "test/fixtures/linter/indentation_errors.sql"
+    fpath2 = "test/fixtures/linter/multiple_sql_errors.sql"
 
     cmd_args = (
-        fpath,
-        fpath,
+        fpath1,
+        fpath2,
         "--format",
         serialize,
         "--disable-progress-bar",
@@ -1313,7 +1314,7 @@ def test__cli__command_lint_serialize_multiple_files(serialize, write_file, tmp_
     print("## End Payload")
 
     if serialize == "human":
-        assert payload_length == 31 if write_file else 32
+        assert payload_length == 26 if write_file else 32
     elif serialize == "json":
         result = json.loads(result_payload)
         assert len(result) == 2
@@ -1323,13 +1324,13 @@ def test__cli__command_lint_serialize_multiple_files(serialize, write_file, tmp_
     elif serialize == "github-annotation":
         result = json.loads(result_payload)
         filepaths = {r["file"] for r in result}
-        assert len(filepaths) == 1
+        assert len(filepaths) == 2
     elif serialize == "github-annotation-native":
         result = result_payload.split("\n")
         # SQLFluff produces trailing newline
         if result[-1] == "":
             del result[-1]
-        assert len(result) == 24
+        assert len(result) == 17
     else:
         raise Exception
 
@@ -1721,8 +1722,16 @@ def test_cli_lint_enabled_progress_bar_multiple_paths(
         )
         raw_output = repr(result.output)
 
-        assert r"\rpath test/fixtures/linter/passing.sql:" in raw_output
-        assert r"\rpath test/fixtures/linter/indentation_errors.sql:" in raw_output
+        sep = os.sep
+        if sys.platform == "win32":
+            sep *= 2
+        assert (
+            r"\rfile test/fixtures/linter/passing.sql:".replace("/", sep) in raw_output
+        )
+        assert (
+            r"\rfile test/fixtures/linter/indentation_errors.sql:".replace("/", sep)
+            in raw_output
+        )
         assert r"\rlint by rules:" in raw_output
         assert r"\rrule L001:" in raw_output
         assert r"\rrule L049:" in raw_output
@@ -1741,9 +1750,27 @@ def test_cli_lint_enabled_progress_bar_multiple_files(
         )
         raw_output = repr(result.output)
 
-        assert r"\rfile passing.1.sql:" in raw_output
-        assert r"\rfile passing.2.sql:" in raw_output
-        assert r"\rfile passing.3.sql:" in raw_output
+        sep = os.sep
+        if sys.platform == "win32":
+            sep *= 2
+        assert (
+            r"\rfile test/fixtures/linter/multiple_files/passing.1.sql:".replace(
+                "/", sep
+            )
+            in raw_output
+        )
+        assert (
+            r"\rfile test/fixtures/linter/multiple_files/passing.2.sql:".replace(
+                "/", sep
+            )
+            in raw_output
+        )
+        assert (
+            r"\rfile test/fixtures/linter/multiple_files/passing.3.sql:".replace(
+                "/", sep
+            )
+            in raw_output
+        )
         assert r"\rlint by rules:" in raw_output
         assert r"\rrule L001:" in raw_output
         assert r"\rrule L049:" in raw_output