Repository: conan-io/conan
Pull request: More fine-grained control (using [conf]) for build parallelization
Changelog: Feature: More fine-grained control (using [conf]) for build parallelization.
Docs: https://github.com/conan-io/docs/pull/2061

This PR introduces the following entries in `conf`:
 * `tools.build:processes`: number of processes to use for every build-helper
 * `tools.gnu.make:jobs`: argument for the `--jobs` parameter when running `make` (overrides the general `tools.build:processes`).
 * `tools.microsoft.msbuild:max_cpu_count`: argument for the `/m` (`/maxCpuCount`) when running `MSBuild` standalone or via CMake (overrides the general `tools.build:processes`).
 * `tools.ninja:jobs`: argument for the `--jobs` parameter when running Ninja generator via CMake or Meson. (overrides the general `tools.build:processes`). 

If none of these configuration options is found, Conan won't add anything to the command line running these tools, it will use whatever it is the default.

close https://github.com/conan-io/conan/issues/8598

Problem statement:
Propose ``cpu_count`` alternative in [conf] for build systems paralelism
For build helpers in ``conan.tools.xxxx``, the parallelism should be configurable via [conf] (and not ``cpu_count``).

Please propose a solution, some guidelines:

- No need that recipes specify it, not explicitly, not as arguments, and they can default back to whatever the build system likes.
- All build systems can share a "parallel" value, no need to do one per build-system/helper
- Might be different to concurrent upload/download

Hints from issue discussion:
What about following entries in `conf`:
 * `core.build:processes`: number of processes to use for every build-helper
 * `tools.gnu.make:jobs`: argument for the `--jobs` parameter when running `make` (overrides the general `core.build:processes`).
 * `tools.microsoft.msbuild:maxCpuCount`: argument for the `/m` (`/maxCpuCount`) when running `MSBuild` standalone or via CMake (overrides the general `core.build:processes`).
 * `tools.ninja:jobs`: argument for the `--jobs` parameter when running Ninja generator via CMake or Meson. (overrides the general `core.build:processes`). If no-parallel is requested, this will force `-j1`. 

If none of these configuration options is found, it fallback to the old `cpu_count` when parallelization is activated. Alternatively, we could leave the number blank (`-j`, `/m`,..) and the corresponding build-system will run in parallel with its internal defaults (infinite probably).

Commit message: add unittesting

Patch to review:
diff --git a/conan/tools/cmake/cmake.py b/conan/tools/cmake/cmake.py
index 049926a0e2c..43c216c91a3 100644
--- a/conan/tools/cmake/cmake.py
+++ b/conan/tools/cmake/cmake.py
@@ -3,13 +3,15 @@
 
 from conan.tools.cmake.base import CMakeToolchainBase
 from conan.tools.cmake.utils import get_generator, is_multi_configuration
-from conan.tools.microsoft.msbuild import msbuild_verbosity_cmd_line_arg
+from conan.tools.gnu.make import make_jobs_cmd_line_arg
+from conan.tools.meson.meson import ninja_jobs_cmd_line_arg
+from conan.tools.microsoft.msbuild import msbuild_verbosity_cmd_line_arg, \
+    msbuild_max_cpu_count_cmd_line_arg
 from conans.client import tools
 from conans.client.build import join_arguments
 from conans.client.tools.files import chdir
 from conans.client.tools.oss import cpu_count, args_to_string
 from conans.errors import ConanException
-from conans.model.version import Version
 from conans.util.conan_v2_mode import conan_v2_error
 from conans.util.files import mkdir
 
@@ -23,15 +25,27 @@ def _validate_recipe(conanfile):
 
 def _cmake_cmd_line_args(conanfile, generator, parallel):
     args = []
-    compiler_version = conanfile.settings.get_safe("compiler.version")
-    if generator and parallel:
-        if ("Makefiles" in generator or "Ninja" in generator) and "NMake" not in generator:
-            args.append("-j%i" % cpu_count(conanfile.output))
-        elif "Visual Studio" in generator and compiler_version and Version(compiler_version) >= "10":
-            # Parallel for building projects in the solution
-            args.append("/m:%i" % cpu_count(output=conanfile.output))
-
-    if generator and "Visual Studio" in generator:
+    if not generator:
+        return args
+
+    # Arguments related to parallel
+    if "Makefiles" in generator and "NMake" not in generator and parallel:
+        njobs = make_jobs_cmd_line_arg(conanfile)
+        if njobs:
+            args.append(njobs)
+
+    if "Ninja" in generator and "NMake" not in generator and parallel:
+        njobs = ninja_jobs_cmd_line_arg(conanfile)
+        if njobs:
+            args.append(njobs)
+
+    if "Visual Studio" in generator and parallel:
+        max_cpu_count = msbuild_max_cpu_count_cmd_line_arg(conanfile)
+        if max_cpu_count:
+            args.append(max_cpu_count)
+
+    # Arguments for verbosity
+    if "Visual Studio" in generator:
         verbosity = msbuild_verbosity_cmd_line_arg(conanfile)
         if verbosity:
             args.append(verbosity)
diff --git a/conan/tools/gnu/make.py b/conan/tools/gnu/make.py
index 7091ad945c5..44fa240d716 100644
--- a/conan/tools/gnu/make.py
+++ b/conan/tools/gnu/make.py
@@ -3,11 +3,19 @@
 from collections import OrderedDict
 
 from jinja2 import Template
+
 from conans.client.build.compiler_flags import build_type_define, libcxx_define
 from conans.client.tools.oss import detected_architecture, detected_os, get_build_os_arch
 from conans.util.files import save
 
 
+def make_jobs_cmd_line_arg(conanfile):
+    njobs = conanfile.conf["tools.gnu.make"].jobs or \
+            conanfile.conf["tools.build"].processes
+    if njobs:
+        return "-j{}".format(njobs)
+
+
 class MakeToolchain(object):
     filename = "conan_toolchain.mak"
 
diff --git a/conan/tools/meson/meson.py b/conan/tools/meson/meson.py
index 8a85db4a569..9c8d98933c4 100644
--- a/conan/tools/meson/meson.py
+++ b/conan/tools/meson/meson.py
@@ -2,7 +2,14 @@
 
 from conan.tools.meson import MesonToolchain
 from conan.tools.microsoft.visual import vcvars_command, vcvars_arch
-from conans.client.tools.oss import cross_building, cpu_count
+from conans.client.tools.oss import cross_building
+
+
+def ninja_jobs_cmd_line_arg(conanfile):
+    njobs = conanfile.conf["tools.ninja"].jobs or \
+            conanfile.conf["tools.build"].processes
+    if njobs:
+        return "-j{}".format(njobs)
 
 
 class Meson(object):
@@ -33,14 +40,17 @@ def configure(self, source_folder=None):
         if cross_building(self._conanfile):
             cmd += ' --cross-file "{}"'.format(MesonToolchain.cross_filename)
         else:
-            cmd += ' --native-file "{}"'. format(MesonToolchain.native_filename)
+            cmd += ' --native-file "{}"'.format(MesonToolchain.native_filename)
         cmd += ' "{}" "{}"'.format(self._build_dir, source)
         if self._conanfile.package_folder:
             cmd += ' -Dprefix="{}"'.format(self._conanfile.package_folder)
         self._run(cmd)
 
     def build(self, target=None):
-        cmd = 'meson compile -C "{}" -j {}'.format(self._build_dir, cpu_count())
+        cmd = 'meson compile -C "{}"'.format(self._build_dir)
+        njobs = ninja_jobs_cmd_line_arg(self._conanfile)
+        if njobs:
+            cmd += " {}".format(njobs)
         if target:
             cmd += " {}".format(target)
         self._run(cmd)
diff --git a/conan/tools/microsoft/msbuild.py b/conan/tools/microsoft/msbuild.py
index d1c22d064b2..3ac28f0c5a8 100644
--- a/conan/tools/microsoft/msbuild.py
+++ b/conan/tools/microsoft/msbuild.py
@@ -9,6 +9,13 @@ def msbuild_verbosity_cmd_line_arg(conanfile):
         return '/verbosity:{}'.format(verbosity)
 
 
+def msbuild_max_cpu_count_cmd_line_arg(conanfile):
+    max_cpu_count = conanfile.conf["tools.microsoft.msbuild"].max_cpu_count or \
+                    conanfile.conf["tools.build"].processes
+    if max_cpu_count:
+        return "/m:{}".format(max_cpu_count)
+
+
 class MSBuild(object):
     def __init__(self, conanfile):
         self._conanfile = conanfile
@@ -34,6 +41,10 @@ def command(self, sln):
         if verbosity:
             cmd += " {}".format(verbosity)
 
+        max_cpu_count = msbuild_max_cpu_count_cmd_line_arg(self._conanfile)
+        if max_cpu_count:
+            cmd += " {}".format(max_cpu_count)
+
         return cmd
 
     def build(self, sln):
diff --git a/conans/test/unittests/tools/cmake/test_cmake_cmd_line_args.py b/conans/test/unittests/tools/cmake/test_cmake_cmd_line_args.py
new file mode 100644
index 00000000000..ba40251dc7e
--- /dev/null
+++ b/conans/test/unittests/tools/cmake/test_cmake_cmd_line_args.py
@@ -0,0 +1,54 @@
+import textwrap
+
+import pytest
+
+from conan.tools.cmake.cmake import _cmake_cmd_line_args
+from conans.model.conf import ConfDefinition
+from conans.test.utils.mocks import ConanFileMock
+
+
+@pytest.fixture
+def conanfile():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+        tools.gnu.make:jobs=40
+        tools.ninja:jobs=30
+        tools.microsoft.msbuild:max_cpu_count=20
+        tools.build:processes=10
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    return conanfile
+
+
+def test_no_generator(conanfile):
+    args = _cmake_cmd_line_args(conanfile, None, parallel=True)
+    assert not len(args)
+
+
+def test_makefiles(conanfile):
+    args = _cmake_cmd_line_args(conanfile, 'Unix Makefiles', parallel=True)
+    assert args == ['-j40']
+
+    args = _cmake_cmd_line_args(conanfile, 'Unix Makefiles', parallel=False)
+    assert not len(args)
+
+    args = _cmake_cmd_line_args(conanfile, 'NMake Makefiles', parallel=True)
+    assert not len(args)
+
+
+def test_ninja(conanfile):
+    args = _cmake_cmd_line_args(conanfile, 'Ninja', parallel=True)
+    assert ['-j30'] == args
+
+    args = _cmake_cmd_line_args(conanfile, 'Ninja', parallel=False)
+    assert not len(args)
+
+
+def test_visual_studio(conanfile):
+    args = _cmake_cmd_line_args(conanfile, 'Visual Studio 16 2019', parallel=True)
+    assert ['/m:20'] == args
+
+    args = _cmake_cmd_line_args(conanfile, 'Ninja', parallel=False)
+    assert not len(args)
diff --git a/conans/test/unittests/tools/gnu/__init__.py b/conans/test/unittests/tools/gnu/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/conans/test/unittests/tools/gnu/test_make_jobs_cmd_line_arg.py b/conans/test/unittests/tools/gnu/test_make_jobs_cmd_line_arg.py
new file mode 100644
index 00000000000..b2a463641c3
--- /dev/null
+++ b/conans/test/unittests/tools/gnu/test_make_jobs_cmd_line_arg.py
@@ -0,0 +1,53 @@
+import textwrap
+
+from conan.tools.gnu.make import make_jobs_cmd_line_arg
+from conans.model.conf import ConfDefinition
+from conans.test.utils.mocks import ConanFileMock
+
+
+def test_tools_build():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+        tools.build:processes=10
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    njobs = make_jobs_cmd_line_arg(conanfile)
+    assert njobs == "-j10"
+
+
+def test_tools_gnu_make():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+        tools.gnu.make:jobs=23
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    njobs = make_jobs_cmd_line_arg(conanfile)
+    assert njobs == "-j23"
+
+
+def test_both_values():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+        tools.gnu.make:jobs=23
+        tools.build:processes=10
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    njobs = make_jobs_cmd_line_arg(conanfile)
+    assert njobs == "-j23"
+
+
+def test_none():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    njobs = make_jobs_cmd_line_arg(conanfile)
+    assert njobs is None
diff --git a/conans/test/unittests/tools/meson/__init__.py b/conans/test/unittests/tools/meson/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/conans/test/unittests/tools/meson/test_meson.py b/conans/test/unittests/tools/meson/test_meson.py
new file mode 100644
index 00000000000..364157f479d
--- /dev/null
+++ b/conans/test/unittests/tools/meson/test_meson.py
@@ -0,0 +1,28 @@
+import textwrap
+
+from conan.tools.meson import Meson
+from conans.model.conf import ConfDefinition
+from conans.test.utils.mocks import ConanFileMock, MockSettings
+
+
+def test_meson_build():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+        tools.ninja:jobs=23
+        tools.build:processes=10
+    """))
+
+    settings = MockSettings({"build_type": "Release",
+                             "compiler": "gcc",
+                             "compiler.version": "7",
+                             "os": "Linux",
+                             "arch": "x86_64"})
+    conanfile = ConanFileMock()
+    conanfile.settings = settings
+    conanfile.display_name = 'test'
+    conanfile.conf = c.get_conanfile_conf(None)
+
+    meson = Meson(conanfile)
+    meson.build()
+    
+    assert '-j23' in str(conanfile.command)
diff --git a/conans/test/unittests/tools/meson/test_ninja_jobs_cmd_line_arg.py b/conans/test/unittests/tools/meson/test_ninja_jobs_cmd_line_arg.py
new file mode 100644
index 00000000000..278f50d8737
--- /dev/null
+++ b/conans/test/unittests/tools/meson/test_ninja_jobs_cmd_line_arg.py
@@ -0,0 +1,53 @@
+import textwrap
+
+from conan.tools.meson.meson import ninja_jobs_cmd_line_arg
+from conans.model.conf import ConfDefinition
+from conans.test.utils.mocks import ConanFileMock
+
+
+def test_tools_build():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+        tools.build:processes=10
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    njobs = ninja_jobs_cmd_line_arg(conanfile)
+    assert njobs == "-j10"
+
+
+def test_tools_ning():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+        tools.ninja:jobs=23
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    njobs = ninja_jobs_cmd_line_arg(conanfile)
+    assert njobs == "-j23"
+
+
+def test_both_values():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+        tools.ninja:jobs=23
+        tools.build:processes=10
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    njobs = ninja_jobs_cmd_line_arg(conanfile)
+    assert njobs == "-j23"
+
+
+def test_none():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    njobs = ninja_jobs_cmd_line_arg(conanfile)
+    assert njobs is None
diff --git a/conans/test/unittests/tools/microsoft/__init__.py b/conans/test/unittests/tools/microsoft/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/conans/test/unittests/tools/microsoft/test_msbuild.py b/conans/test/unittests/tools/microsoft/test_msbuild.py
new file mode 100644
index 00000000000..28bbdcf94a2
--- /dev/null
+++ b/conans/test/unittests/tools/microsoft/test_msbuild.py
@@ -0,0 +1,27 @@
+import textwrap
+
+from conan.tools.microsoft import MSBuild
+from conans.model.conf import ConfDefinition
+from conans.test.utils.mocks import ConanFileMock, MockSettings
+
+
+def test_meson_build():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+        tools.microsoft.msbuild:max_cpu_count=23
+        tools.build:processes=10
+    """))
+
+    settings = MockSettings({"build_type": "Release",
+                             "compiler": "gcc",
+                             "compiler.version": "7",
+                             "os": "Linux",
+                             "arch": "x86_64"})
+    conanfile = ConanFileMock()
+    conanfile.settings = settings
+    conanfile.conf = c.get_conanfile_conf(None)
+
+    msbuild = MSBuild(conanfile)
+    cmd = msbuild.command('project.sln')
+
+    assert '/m:23' in cmd
diff --git a/conans/test/unittests/tools/microsoft/test_msbuild_max_cpu_count_cmd_line_arg.py b/conans/test/unittests/tools/microsoft/test_msbuild_max_cpu_count_cmd_line_arg.py
new file mode 100644
index 00000000000..95fd0452a5b
--- /dev/null
+++ b/conans/test/unittests/tools/microsoft/test_msbuild_max_cpu_count_cmd_line_arg.py
@@ -0,0 +1,53 @@
+import textwrap
+
+from conan.tools.microsoft.msbuild import msbuild_max_cpu_count_cmd_line_arg
+from conans.model.conf import ConfDefinition
+from conans.test.utils.mocks import ConanFileMock
+
+
+def test_tools_build():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+        tools.build:processes=10
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    max_cpu_count = msbuild_max_cpu_count_cmd_line_arg(conanfile)
+    assert max_cpu_count == "/m:10"
+
+
+def test_tools_ning():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+        tools.microsoft.msbuild:max_cpu_count=23
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    max_cpu_count = msbuild_max_cpu_count_cmd_line_arg(conanfile)
+    assert max_cpu_count == "/m:23"
+
+
+def test_both_values():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+        tools.microsoft.msbuild:max_cpu_count=23
+        tools.build:processes=10
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    max_cpu_count = msbuild_max_cpu_count_cmd_line_arg(conanfile)
+    assert max_cpu_count == "/m:23"
+
+
+def test_none():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    max_cpu_count = msbuild_max_cpu_count_cmd_line_arg(conanfile)
+    assert max_cpu_count is None


Code review comments:
diff_hunk: @@ -23,15 +25,27 @@ def _validate_recipe(conanfile):
 
 def _cmake_cmd_line_args(conanfile, generator, parallel):
     args = []
-    compiler_version = conanfile.settings.get_safe("compiler.version")
-    if generator and parallel:
-        if ("Makefiles" in generator or "Ninja" in generator) and "NMake" not in generator:
-            args.append("-j%i" % cpu_count(conanfile.output))
-        elif "Visual Studio" in generator and compiler_version and Version(compiler_version) >= "10":
-            # Parallel for building projects in the solution
-            args.append("/m:%i" % cpu_count(output=conanfile.output))
-
-    if generator and "Visual Studio" in generator:
+    if not generator:
+        return args
+
+    # Arguments related to parallel
+    if "Makefiles" in generator and "NMake" not in generator and parallel:
path: conan/tools/cmake/cmake.py
text: @user1:
maybe check for parallel just once and use nested cond? we're checking 3 times right now