Repository: mesonbuild/meson

Issue:
Python extension_module does not install in site_packages
In the *python* module documentation is claimed that `extension_module` install per default in `site_packages`:
> `subdir`: By default, meson will install the extension module in the relevant top-level location for the python installation, eg `/usr/lib/site-packages`. When subdir is passed to this method, it will be appended to that location. This keyword argument is mutually exclusive with `install_dir`

However, this does not work. When set `install: true` the modules are installed to `$prefix/lib64` on my system.

One workaround is to specify the target like:
```
py3_mod = import('python')
py3_inst = py3_mod.find_installation('python3')

foo = py3_inst.extension_module('foo',
    <source_target>,
    install: true,
    subdir: get_option('prefix') + py3_inst.get_path('purelib'),
    dependencies: [py3_inst.dependency()]
)
```
It would be nice, if the subdir is set autotically to the `site-packages` folder.

This bug is btw similar to #2859. However, this talks about the now deprecated *python3* module.



Discussion:
Same here, except since I build a C module, I need to use `install_dir: py3_inst.get_path('platlib')`

Resolving patch:
diff --git a/mesonbuild/modules/python.py b/mesonbuild/modules/python.py
--- a/mesonbuild/modules/python.py
+++ b/mesonbuild/modules/python.py
@@ -306,6 +306,7 @@ class PythonIntrospectionDict(TypedDict):
         link_libpython: bool
         paths: T.Dict[str, str]
         platform: str
+        suffix : str
         variables: T.Dict[str, str]
         version: str
 
@@ -316,7 +317,7 @@ def __init__(self, name: str, command: T.Optional[T.List[str]] = None,
         if ext_prog is None:
             super().__init__(name, command=command, silent=True)
         else:
-            self.name = ext_prog.name
+            self.name = name
             self.command = ext_prog.command
             self.path = ext_prog.path
 
@@ -334,6 +335,35 @@ def __init__(self, name: str, command: T.Optional[T.List[str]] = None,
             'version': '0.0',
         }
 
+    def _check_version(self, version: str) -> bool:
+        if self.name == 'python2':
+            return mesonlib.version_compare(version, '< 3.0')
+        elif self.name == 'python3':
+            return mesonlib.version_compare(version, '>= 3.0')
+        return True
+
+    def sanity(self) -> bool:
+        # Sanity check, we expect to have something that at least quacks in tune
+        cmd = self.get_command() + ['-c', INTROSPECT_COMMAND]
+        p, stdout, stderr = mesonlib.Popen_safe(cmd)
+        try:
+            info = json.loads(stdout)
+        except json.JSONDecodeError:
+            info = None
+            mlog.debug('Could not introspect Python (%s): exit code %d' % (str(p.args), p.returncode))
+            mlog.debug('Program stdout:\n')
+            mlog.debug(stdout)
+            mlog.debug('Program stderr:\n')
+            mlog.debug(stderr)
+
+        if info is not None and self._check_version(info['version']):
+            variables = info['variables']
+            info['suffix'] = variables.get('EXT_SUFFIX') or variables.get('SO') or variables.get('.so')
+            self.info = T.cast('PythonIntrospectionDict', info)
+            return True
+        else:
+            return False
+
 
 _PURE_KW = KwargInfo('pure', bool, default=True)
 _SUBDIR_KW = KwargInfo('subdir', str, default='')
@@ -354,6 +384,7 @@ def __init__(self, python: 'PythonExternalProgram', interpreter: 'Interpreter'):
         prefix = self.interpreter.environment.coredata.get_option(mesonlib.OptionKey('prefix'))
         assert isinstance(prefix, str), 'for mypy'
         self.variables = info['variables']
+        self.suffix = info['suffix']
         self.paths = info['paths']
         install_paths = info['install_paths']
         self.platlib_install_path = os.path.join(prefix, install_paths['platlib'][1:])
@@ -378,10 +409,10 @@ def __init__(self, python: 'PythonExternalProgram', interpreter: 'Interpreter'):
 
     @permittedKwargs(mod_kwargs)
     def extension_module_method(self, args: T.List['TYPE_var'], kwargs: 'TYPE_kwargs') -> 'SharedModule':
-        if 'subdir' in kwargs and 'install_dir' in kwargs:
-            raise InvalidArguments('"subdir" and "install_dir" are mutually exclusive')
-
-        if 'subdir' in kwargs:
+        if 'install_dir' in kwargs:
+            if 'subdir' in kwargs:
+                raise InvalidArguments('"subdir" and "install_dir" are mutually exclusive')
+        else:
             subdir = kwargs.pop('subdir', '')
             if not isinstance(subdir, str):
                 raise InvalidArguments('"subdir" argument must be a string.')
@@ -399,12 +430,10 @@ def extension_module_method(self, args: T.List['TYPE_var'], kwargs: 'TYPE_kwargs
                 new_deps.append(dep)
             kwargs['dependencies'] = new_deps
 
-        suffix = self.variables.get('EXT_SUFFIX') or self.variables.get('SO') or self.variables.get('.so')
-
         # msys2's python3 has "-cpython-36m.dll", we have to be clever
-        split = suffix.rsplit('.', 1)
-        suffix = split.pop(-1)
-        args[0] += ''.join(s for s in split)
+        # FIXME: explain what the specific cleverness is here
+        split, suffix = self.suffix.rsplit('.', 1)
+        args[0] += split
 
         kwargs['name_prefix'] = ''
         kwargs['name_suffix'] = suffix
@@ -535,14 +564,6 @@ def _get_win_pythonpath(name_or_path: str) -> T.Optional[str]:
         else:
             return None
 
-    @staticmethod
-    def _check_version(name_or_path: str, version: str) -> bool:
-        if name_or_path == 'python2':
-            return mesonlib.version_compare(version, '< 3.0')
-        elif name_or_path == 'python3':
-            return mesonlib.version_compare(version, '>= 3.0')
-        return True
-
     @disablerIfNotFound
     @typed_pos_args('python.find_installation', optargs=[str])
     @typed_kwargs(
@@ -567,6 +588,7 @@ def find_installation(self, state: 'ModuleState', args: T.Tuple[T.Optional[str]]
         # $PATH, or that uses a wrapper of some kind.
         np: T.List[str] = state.environment.lookup_binary_entry(MachineChoice.HOST, 'python') or []
         fallback = args[0]
+        display_name = fallback or 'python'
         if not np and fallback is not None:
             np = [fallback]
         name_or_path = np[0] if np else None
@@ -578,8 +600,8 @@ def find_installation(self, state: 'ModuleState', args: T.Tuple[T.Optional[str]]
         if not name_or_path:
             python = PythonExternalProgram('python3', mesonlib.python_command)
         else:
-            tmp_python = ExternalProgram.from_entry('python3', name_or_path)
-            python = PythonExternalProgram('python3', ext_prog=tmp_python)
+            tmp_python = ExternalProgram.from_entry(display_name, name_or_path)
+            python = PythonExternalProgram(display_name, ext_prog=tmp_python)
 
             if not python.found() and mesonlib.is_windows():
                 pythonpath = self._get_win_pythonpath(name_or_path)
@@ -627,21 +649,9 @@ def find_installation(self, state: 'ModuleState', args: T.Tuple[T.Optional[str]]
                 raise mesonlib.MesonException('{} is missing modules: {}'.format(name_or_path or 'python', ', '.join(missing_modules)))
             return NonExistingExternalProgram()
         else:
-            # Sanity check, we expect to have something that at least quacks in tune
-            cmd = python.get_command() + ['-c', INTROSPECT_COMMAND]
-            p, stdout, stderr = mesonlib.Popen_safe(cmd)
-            try:
-                info = json.loads(stdout)
-            except json.JSONDecodeError:
-                info = None
-                mlog.debug('Could not introspect Python (%s): exit code %d' % (str(p.args), p.returncode))
-                mlog.debug('Program stdout:\n')
-                mlog.debug(stdout)
-                mlog.debug('Program stderr:\n')
-                mlog.debug(stderr)
-
-            if isinstance(info, dict) and 'version' in info and self._check_version(name_or_path, info['version']):
-                python.info = T.cast('PythonIntrospectionDict', info)
+            sane = python.sanity()
+
+            if sane:
                 return python
             else:
                 if required:


Test patch:
diff --git a/data/test.schema.json b/data/test.schema.json
--- a/data/test.schema.json
+++ b/data/test.schema.json
@@ -21,11 +21,14 @@
             "type": "string",
             "enum": [
               "file",
+              "python_file",
               "dir",
               "exe",
               "shared_lib",
+              "python_lib",
               "pdb",
               "implib",
+              "py_implib",
               "implibempty",
               "expr"
             ]
diff --git a/run_project_tests.py b/run_project_tests.py
--- a/run_project_tests.py
+++ b/run_project_tests.py
@@ -52,6 +52,7 @@
 from mesonbuild.mlog import blue, bold, cyan, green, red, yellow, normal_green
 from mesonbuild.coredata import backendlist, version as meson_version
 from mesonbuild.mesonmain import setup_vsenv
+from mesonbuild.modules.python import PythonExternalProgram
 from run_tests import get_fake_options, run_configure, get_meson_script
 from run_tests import get_backend_commands, get_backend_args_for_dir, Backend
 from run_tests import ensure_backend_detects_changes
@@ -62,6 +63,7 @@
     from mesonbuild.environment import Environment
     from mesonbuild._typing import Protocol
     from concurrent.futures import Future
+    from mesonbuild.modules.python import PythonIntrospectionDict
 
     class CompilerArgumentType(Protocol):
         cross_file: str
@@ -122,6 +124,9 @@ def add_step(self, step: BuildStep, stdo: str, stde: str, mlog: str = '', time:
     def fail(self, msg: str) -> None:
         self.msg = msg
 
+python = PythonExternalProgram(sys.executable)
+python.sanity()
+
 class InstalledFile:
     def __init__(self, raw: T.Dict[str, str]):
         self.path = raw['file']
@@ -143,6 +148,9 @@ def get_path(self, compiler: str, env: environment.Environment) -> T.Optional[Pa
                 (env.machines.host.is_windows() and compiler in {'pgi', 'dmd', 'ldc'})):
             canonical_compiler = 'msvc'
 
+        python_paths = python.info['install_paths']
+        python_suffix = python.info['suffix']
+
         has_pdb = False
         if self.language in {'c', 'cpp'}:
             has_pdb = canonical_compiler == 'msvc'
@@ -161,6 +169,15 @@ def get_path(self, compiler: str, env: environment.Environment) -> T.Optional[Pa
             return None
 
         # Handle the different types
+        if self.typ in {'py_implib', 'python_lib', 'python_file'}:
+            val = p.as_posix()
+            val = val.replace('@PYTHON_PLATLIB@', python_paths['platlib'])
+            val = val.replace('@PYTHON_PURELIB@', python_paths['purelib'])
+            p = Path(val)
+            if self.typ == 'python_file':
+                return p
+            if self.typ == 'python_lib':
+                return p.with_suffix(python_suffix)
         if self.typ in ['file', 'dir']:
             return p
         elif self.typ == 'shared_lib':
@@ -195,13 +212,15 @@ def get_path(self, compiler: str, env: environment.Environment) -> T.Optional[Pa
             if self.version:
                 p = p.with_name('{}-{}'.format(p.name, self.version[0]))
             return p.with_suffix('.pdb') if has_pdb else None
-        elif self.typ == 'implib' or self.typ == 'implibempty':
+        elif self.typ in {'implib', 'implibempty', 'py_implib'}:
             if env.machines.host.is_windows() and canonical_compiler == 'msvc':
                 # only MSVC doesn't generate empty implibs
                 if self.typ == 'implibempty' and compiler == 'msvc':
                     return None
                 return p.parent / (re.sub(r'^lib', '', p.name) + '.lib')
             elif env.machines.host.is_windows() or env.machines.host.is_cygwin():
+                if self.typ == 'py_implib':
+                    p = p.with_suffix(python_suffix)
                 return p.with_suffix('.dll.a')
             else:
                 return None
diff --git a/test cases/python/2 extmodule/blaster.py b/test cases/python/2 extmodule/blaster.py.in
similarity index 86%
rename from test cases/python/2 extmodule/blaster.py
rename to test cases/python/2 extmodule/blaster.py.in
--- a/test cases/python/2 extmodule/blaster.py	
+++ b/test cases/python/2 extmodule/blaster.py.in	
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-import tachyon
+import @tachyon_module@ as tachyon
 
 result = tachyon.phaserize('shoot')
 
diff --git a/test cases/python/2 extmodule/ext/meson.build b/test cases/python/2 extmodule/ext/meson.build
--- a/test cases/python/2 extmodule/ext/meson.build	
+++ b/test cases/python/2 extmodule/ext/meson.build	
@@ -1,6 +1,10 @@
 pylib = py.extension_module('tachyon',
   'tachyon_module.c',
   dependencies : py_dep,
+  c_args: '-DMESON_MODULENAME="tachyon"',
+  install: true,
 )
 
+subdir('nested')
+subdir('wrongdir')
 pypathdir = meson.current_build_dir()
diff --git a/test cases/python/2 extmodule/ext/nested/meson.build b/test cases/python/2 extmodule/ext/nested/meson.build
new file mode 100644
--- /dev/null
+++ b/test cases/python/2 extmodule/ext/nested/meson.build	
@@ -0,0 +1,16 @@
+py.extension_module('tachyon',
+  '../tachyon_module.c',
+  dependencies : py_dep,
+  c_args: '-DMESON_MODULENAME="nested.tachyon"',
+  install: true,
+  subdir: 'nested'
+)
+py.install_sources(
+  configure_file(
+    input: '../../blaster.py.in',
+    output: 'blaster.py',
+    configuration: {'tachyon_module': 'nested.tachyon'}
+  ),
+  pure: false,
+  subdir: 'nested',
+)
diff --git a/test cases/python/2 extmodule/ext/tachyon_module.c b/test cases/python/2 extmodule/ext/tachyon_module.c
--- a/test cases/python/2 extmodule/ext/tachyon_module.c	
+++ b/test cases/python/2 extmodule/ext/tachyon_module.c	
@@ -38,7 +38,7 @@ static PyMethodDef TachyonMethods[] = {
 
 static struct PyModuleDef tachyonmodule = {
    PyModuleDef_HEAD_INIT,
-   "tachyon",
+   MESON_MODULENAME,
    NULL,
    -1,
    TachyonMethods
diff --git a/test cases/python/2 extmodule/ext/wrongdir/meson.build b/test cases/python/2 extmodule/ext/wrongdir/meson.build
new file mode 100644
--- /dev/null
+++ b/test cases/python/2 extmodule/ext/wrongdir/meson.build	
@@ -0,0 +1,7 @@
+py.extension_module('tachyon',
+  '../tachyon_module.c',
+  dependencies : py_dep,
+  c_args: '-DMESON_MODULENAME="tachyon"',
+  install: true,
+  install_dir: get_option('libdir')
+)
diff --git a/test cases/python/2 extmodule/meson.build b/test cases/python/2 extmodule/meson.build
--- a/test cases/python/2 extmodule/meson.build	
+++ b/test cases/python/2 extmodule/meson.build	
@@ -18,11 +18,19 @@ endif
 
 subdir('ext')
 
+blaster = configure_file(
+  input: 'blaster.py.in',
+  output: 'blaster.py',
+  configuration: {'tachyon_module': 'tachyon'}
+)
+
 test('extmod',
   py,
-  args : files('blaster.py'),
+  args : blaster,
   env : ['PYTHONPATH=' + pypathdir])
 
+py.install_sources(blaster, pure: false)
+py.install_sources(blaster, subdir: 'pure')
 
 py3_pkg_dep = dependency('python3', method: 'pkg-config', required : false)
 if py3_pkg_dep.found()
diff --git a/test cases/python/2 extmodule/test.json b/test cases/python/2 extmodule/test.json
new file mode 100644
--- /dev/null
+++ b/test cases/python/2 extmodule/test.json	
@@ -0,0 +1,13 @@
+{
+  "installed": [
+    { "type": "python_file", "file": "usr/@PYTHON_PLATLIB@/blaster.py" },
+    { "type": "python_lib",  "file": "usr/@PYTHON_PLATLIB@/tachyon" },
+    { "type": "py_implib",   "file": "usr/@PYTHON_PLATLIB@/tachyon" },
+    { "type": "python_file", "file": "usr/@PYTHON_PURELIB@/pure/blaster.py" },
+    { "type": "python_file", "file": "usr/@PYTHON_PLATLIB@/nested/blaster.py" },
+    { "type": "python_lib",  "file": "usr/@PYTHON_PLATLIB@/nested/tachyon" },
+    { "type": "py_implib",   "file": "usr/@PYTHON_PLATLIB@/nested/tachyon" },
+    { "type": "python_lib",  "file": "usr/lib/tachyon" },
+    { "type": "py_implib",   "file": "usr/lib/tachyon" }
+  ]
+}