Repository: mesonbuild/meson

Issue:
Python extension_module does not install in site_packages
In the *python* module documentation is claimed that `extension_module` install per default in `site_packages`:
> `subdir`: By default, meson will install the extension module in the relevant top-level location for the python installation, eg `/usr/lib/site-packages`. When subdir is passed to this method, it will be appended to that location. This keyword argument is mutually exclusive with `install_dir`

However, this does not work. When set `install: true` the modules are installed to `$prefix/lib64` on my system.

One workaround is to specify the target like:
```
py3_mod = import('python')
py3_inst = py3_mod.find_installation('python3')

foo = py3_inst.extension_module('foo',
    <source_target>,
    install: true,
    subdir: get_option('prefix') + py3_inst.get_path('purelib'),
    dependencies: [py3_inst.dependency()]
)
```
It would be nice, if the subdir is set autotically to the `site-packages` folder.

This bug is btw similar to #2859. However, this talks about the now deprecated *python3* module.



Discussion:
Same here, except since I build a C module, I need to use `install_dir: py3_inst.get_path('platlib')`

Resolving patch:
diff --git a/mesonbuild/modules/python.py b/mesonbuild/modules/python.py
--- a/mesonbuild/modules/python.py
+++ b/mesonbuild/modules/python.py
@@ -279,15 +279,65 @@ def links_against_libpython():
 }))
 '''
 
+if T.TYPE_CHECKING:
+    class PythonIntrospectionDict(TypedDict):
+
+        install_paths: T.Dict[str, str]
+        is_pypy: bool
+        link_libpython: bool
+        paths: T.Dict[str, str]
+        platform: str
+        suffix : str
+        variables: T.Dict[str, str]
+        version: str
+
 class PythonExternalProgram(ExternalProgram):
     def __init__(self, name: str, command: T.Optional[T.List[str]] = None, ext_prog: T.Optional[ExternalProgram] = None):
         if ext_prog is None:
             super().__init__(name, command=command, silent=True)
         else:
-            self.name = ext_prog.name
+            self.name = name
             self.command = ext_prog.command
             self.path = ext_prog.path
-        self.info: T.Dict[str, str] = {}
+        self.info: 'PythonIntrospectionDict' = {
+            'install_paths': {},
+            'is_pypy': False,
+            'link_libpython': False,
+            'paths': {},
+            'platform': 'sentinal',
+            'variables': {},
+            'version': '0.0',
+        }
+
+    def _check_version(self, version: str) -> bool:
+        if self.name == 'python2':
+            return mesonlib.version_compare(version, '< 3.0')
+        elif self.name == 'python3':
+            return mesonlib.version_compare(version, '>= 3.0')
+        return True
+
+    def sanity(self) -> bool:
+        # Sanity check, we expect to have something that at least quacks in tune
+        cmd = self.get_command() + ['-c', INTROSPECT_COMMAND]
+        p, stdout, stderr = mesonlib.Popen_safe(cmd)
+        try:
+            info = json.loads(stdout)
+        except json.JSONDecodeError:
+            info = None
+            mlog.debug('Could not introspect Python (%s): exit code %d' % (str(p.args), p.returncode))
+            mlog.debug('Program stdout:\n')
+            mlog.debug(stdout)
+            mlog.debug('Program stderr:\n')
+            mlog.debug(stderr)
+
+        if info is not None and self._check_version(info['version']):
+            variables = info['variables']
+            info['suffix'] = variables.get('EXT_SUFFIX') or variables.get('SO') or variables.get('.so')
+            self.info = T.cast('PythonIntrospectionDict', info)
+            return True
+        else:
+            return False
+
 
 class PythonInstallation(ExternalProgramHolder):
     def __init__(self, python, interpreter):
@@ -295,6 +345,7 @@ def __init__(self, python, interpreter):
         info = python.info
         prefix = self.interpreter.environment.coredata.get_option(mesonlib.OptionKey('prefix'))
         self.variables = info['variables']
+        self.suffix = info['suffix']
         self.paths = info['paths']
         install_paths = info['install_paths']
         self.platlib_install_path = os.path.join(prefix, install_paths['platlib'][1:])
@@ -319,10 +370,10 @@ def __init__(self, python, interpreter):
 
     @permittedKwargs(mod_kwargs)
     def extension_module_method(self, args, kwargs):
-        if 'subdir' in kwargs and 'install_dir' in kwargs:
-            raise InvalidArguments('"subdir" and "install_dir" are mutually exclusive')
-
-        if 'subdir' in kwargs:
+        if 'install_dir' in kwargs:
+            if 'subdir' in kwargs:
+                raise InvalidArguments('"subdir" and "install_dir" are mutually exclusive')
+        else:
             subdir = kwargs.pop('subdir', '')
             if not isinstance(subdir, str):
                 raise InvalidArguments('"subdir" argument must be a string.')
@@ -340,12 +391,10 @@ def extension_module_method(self, args, kwargs):
                 new_deps.append(dep)
             kwargs['dependencies'] = new_deps
 
-        suffix = self.variables.get('EXT_SUFFIX') or self.variables.get('SO') or self.variables.get('.so')
-
         # msys2's python3 has "-cpython-36m.dll", we have to be clever
-        split = suffix.rsplit('.', 1)
-        suffix = split.pop(-1)
-        args[0] += ''.join(s for s in split)
+        # FIXME: explain what the specific cleverness is here
+        split, suffix = self.suffix.rsplit('.', 1)
+        args[0] += split
 
         kwargs['name_prefix'] = ''
         kwargs['name_suffix'] = suffix
@@ -498,12 +547,6 @@ def _get_win_pythonpath(self, name_or_path):
         else:
             return None
 
-    def _check_version(self, name_or_path, version):
-        if name_or_path == 'python2':
-            return mesonlib.version_compare(version, '< 3.0')
-        elif name_or_path == 'python3':
-            return mesonlib.version_compare(version, '>= 3.0')
-        return True
 
     @FeatureNewKwargs('python.find_installation', '0.49.0', ['disabler'])
     @FeatureNewKwargs('python.find_installation', '0.51.0', ['modules'])
@@ -515,13 +558,15 @@ def find_installation(self, state, args, kwargs):
         want_modules = mesonlib.extract_as_list(kwargs, 'modules')  # type: T.List[str]
         found_modules = []    # type: T.List[str]
         missing_modules = []  # type: T.List[str]
+        fallback = args[0] if args else ''
+        display_name = fallback or 'python'
 
         if len(args) > 1:
             raise InvalidArguments('find_installation takes zero or one positional argument.')
 
         name_or_path = state.environment.lookup_binary_entry(MachineChoice.HOST, 'python')
         if name_or_path is None and args:
-            name_or_path = args[0]
+            name_or_path = fallback
             if not isinstance(name_or_path, str):
                 raise InvalidArguments('find_installation argument must be a string.')
 
@@ -532,8 +577,8 @@ def find_installation(self, state, args, kwargs):
         if not name_or_path:
             python = PythonExternalProgram('python3', mesonlib.python_command)
         else:
-            tmp_python = ExternalProgram.from_entry('python3', name_or_path)
-            python = PythonExternalProgram('python3', ext_prog=tmp_python)
+            tmp_python = ExternalProgram.from_entry(display_name, name_or_path)
+            python = PythonExternalProgram(display_name, ext_prog=tmp_python)
 
             if not python.found() and mesonlib.is_windows():
                 pythonpath = self._get_win_pythonpath(name_or_path)
@@ -581,21 +626,9 @@ def find_installation(self, state, args, kwargs):
                 raise mesonlib.MesonException('{} is missing modules: {}'.format(name_or_path or 'python', ', '.join(missing_modules)))
             return NonExistingExternalProgram()
         else:
-            # Sanity check, we expect to have something that at least quacks in tune
-            try:
-                cmd = python.get_command() + ['-c', INTROSPECT_COMMAND]
-                p, stdout, stderr = mesonlib.Popen_safe(cmd)
-                info = json.loads(stdout)
-            except json.JSONDecodeError:
-                info = None
-                mlog.debug('Could not introspect Python (%s): exit code %d' % (str(p.args), p.returncode))
-                mlog.debug('Program stdout:\n')
-                mlog.debug(stdout)
-                mlog.debug('Program stderr:\n')
-                mlog.debug(stderr)
-
-            if isinstance(info, dict) and 'version' in info and self._check_version(name_or_path, info['version']):
-                python.info = info
+            sane = python.sanity()
+
+            if sane:
                 return python
             else:
                 if required:


Test patch:
diff --git a/data/test.schema.json b/data/test.schema.json
--- a/data/test.schema.json
+++ b/data/test.schema.json
@@ -21,11 +21,14 @@
             "type": "string",
             "enum": [
               "file",
+              "python_file",
               "dir",
               "exe",
               "shared_lib",
+              "python_lib",
               "pdb",
               "implib",
+              "py_implib",
               "implibempty",
               "expr"
             ]
diff --git a/run_project_tests.py b/run_project_tests.py
--- a/run_project_tests.py
+++ b/run_project_tests.py
@@ -52,6 +52,7 @@
 from mesonbuild.mlog import blue, bold, cyan, green, red, yellow, normal_green
 from mesonbuild.coredata import backendlist, version as meson_version
 from mesonbuild.mesonmain import setup_vsenv
+from mesonbuild.modules.python import PythonExternalProgram
 from run_tests import get_fake_options, run_configure, get_meson_script
 from run_tests import get_backend_commands, get_backend_args_for_dir, Backend
 from run_tests import ensure_backend_detects_changes
@@ -122,6 +123,9 @@ def add_step(self, step: BuildStep, stdo: str, stde: str, mlog: str = '', time:
     def fail(self, msg: str) -> None:
         self.msg = msg
 
+python = PythonExternalProgram(sys.executable)
+python.sanity()
+
 class InstalledFile:
     def __init__(self, raw: T.Dict[str, str]):
         self.path = raw['file']
@@ -143,6 +147,9 @@ def get_path(self, compiler: str, env: environment.Environment) -> T.Optional[Pa
                 (env.machines.host.is_windows() and compiler in {'pgi', 'dmd', 'ldc'})):
             canonical_compiler = 'msvc'
 
+        python_paths = python.info['install_paths']
+        python_suffix = python.info['suffix']
+
         has_pdb = False
         if self.language in {'c', 'cpp'}:
             has_pdb = canonical_compiler == 'msvc'
@@ -161,6 +168,15 @@ def get_path(self, compiler: str, env: environment.Environment) -> T.Optional[Pa
             return None
 
         # Handle the different types
+        if self.typ in {'py_implib', 'python_lib', 'python_file'}:
+            val = p.as_posix()
+            val = val.replace('@PYTHON_PLATLIB@', python_paths['platlib'])
+            val = val.replace('@PYTHON_PURELIB@', python_paths['purelib'])
+            p = Path(val)
+            if self.typ == 'python_file':
+                return p
+            if self.typ == 'python_lib':
+                return p.with_suffix(python_suffix)
         if self.typ in ['file', 'dir']:
             return p
         elif self.typ == 'shared_lib':
@@ -195,13 +211,15 @@ def get_path(self, compiler: str, env: environment.Environment) -> T.Optional[Pa
             if self.version:
                 p = p.with_name('{}-{}'.format(p.name, self.version[0]))
             return p.with_suffix('.pdb') if has_pdb else None
-        elif self.typ == 'implib' or self.typ == 'implibempty':
+        elif self.typ in {'implib', 'implibempty', 'py_implib'}:
             if env.machines.host.is_windows() and canonical_compiler == 'msvc':
                 # only MSVC doesn't generate empty implibs
                 if self.typ == 'implibempty' and compiler == 'msvc':
                     return None
                 return p.parent / (re.sub(r'^lib', '', p.name) + '.lib')
             elif env.machines.host.is_windows() or env.machines.host.is_cygwin():
+                if self.typ == 'py_implib':
+                    p = p.with_suffix(python_suffix)
                 return p.with_suffix('.dll.a')
             else:
                 return None
diff --git a/test cases/python/2 extmodule/blaster.py b/test cases/python/2 extmodule/blaster.py.in
similarity index 86%
rename from test cases/python/2 extmodule/blaster.py
rename to test cases/python/2 extmodule/blaster.py.in
--- a/test cases/python/2 extmodule/blaster.py	
+++ b/test cases/python/2 extmodule/blaster.py.in	
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-import tachyon
+import @tachyon_module@ as tachyon
 
 result = tachyon.phaserize('shoot')
 
diff --git a/test cases/python/2 extmodule/ext/meson.build b/test cases/python/2 extmodule/ext/meson.build
--- a/test cases/python/2 extmodule/ext/meson.build	
+++ b/test cases/python/2 extmodule/ext/meson.build	
@@ -1,6 +1,10 @@
 pylib = py.extension_module('tachyon',
   'tachyon_module.c',
   dependencies : py_dep,
+  c_args: '-DMESON_MODULENAME="tachyon"',
+  install: true,
 )
 
+subdir('nested')
+subdir('wrongdir')
 pypathdir = meson.current_build_dir()
diff --git a/test cases/python/2 extmodule/ext/nested/meson.build b/test cases/python/2 extmodule/ext/nested/meson.build
new file mode 100644
--- /dev/null
+++ b/test cases/python/2 extmodule/ext/nested/meson.build	
@@ -0,0 +1,16 @@
+py.extension_module('tachyon',
+  '../tachyon_module.c',
+  dependencies : py_dep,
+  c_args: '-DMESON_MODULENAME="nested.tachyon"',
+  install: true,
+  subdir: 'nested'
+)
+py.install_sources(
+  configure_file(
+    input: '../../blaster.py.in',
+    output: 'blaster.py',
+    configuration: {'tachyon_module': 'nested.tachyon'}
+  ),
+  pure: false,
+  subdir: 'nested',
+)
diff --git a/test cases/python/2 extmodule/ext/tachyon_module.c b/test cases/python/2 extmodule/ext/tachyon_module.c
--- a/test cases/python/2 extmodule/ext/tachyon_module.c	
+++ b/test cases/python/2 extmodule/ext/tachyon_module.c	
@@ -38,7 +38,7 @@ static PyMethodDef TachyonMethods[] = {
 
 static struct PyModuleDef tachyonmodule = {
    PyModuleDef_HEAD_INIT,
-   "tachyon",
+   MESON_MODULENAME,
    NULL,
    -1,
    TachyonMethods
diff --git a/test cases/python/2 extmodule/ext/wrongdir/meson.build b/test cases/python/2 extmodule/ext/wrongdir/meson.build
new file mode 100644
--- /dev/null
+++ b/test cases/python/2 extmodule/ext/wrongdir/meson.build	
@@ -0,0 +1,7 @@
+py.extension_module('tachyon',
+  '../tachyon_module.c',
+  dependencies : py_dep,
+  c_args: '-DMESON_MODULENAME="tachyon"',
+  install: true,
+  install_dir: get_option('libdir')
+)
diff --git a/test cases/python/2 extmodule/meson.build b/test cases/python/2 extmodule/meson.build
--- a/test cases/python/2 extmodule/meson.build	
+++ b/test cases/python/2 extmodule/meson.build	
@@ -18,11 +18,19 @@ endif
 
 subdir('ext')
 
+blaster = configure_file(
+  input: 'blaster.py.in',
+  output: 'blaster.py',
+  configuration: {'tachyon_module': 'tachyon'}
+)
+
 test('extmod',
   py,
-  args : files('blaster.py'),
+  args : blaster,
   env : ['PYTHONPATH=' + pypathdir])
 
+py.install_sources(blaster, pure: false)
+py.install_sources(blaster, subdir: 'pure')
 
 py3_pkg_dep = dependency('python3', method: 'pkg-config', required : false)
 if py3_pkg_dep.found()
diff --git a/test cases/python/2 extmodule/test.json b/test cases/python/2 extmodule/test.json
new file mode 100644
--- /dev/null
+++ b/test cases/python/2 extmodule/test.json	
@@ -0,0 +1,13 @@
+{
+  "installed": [
+    { "type": "python_file", "file": "usr/@PYTHON_PLATLIB@/blaster.py" },
+    { "type": "python_lib",  "file": "usr/@PYTHON_PLATLIB@/tachyon" },
+    { "type": "py_implib",   "file": "usr/@PYTHON_PLATLIB@/tachyon" },
+    { "type": "python_file", "file": "usr/@PYTHON_PURELIB@/pure/blaster.py" },
+    { "type": "python_file", "file": "usr/@PYTHON_PLATLIB@/nested/blaster.py" },
+    { "type": "python_lib",  "file": "usr/@PYTHON_PLATLIB@/nested/tachyon" },
+    { "type": "py_implib",   "file": "usr/@PYTHON_PLATLIB@/nested/tachyon" },
+    { "type": "python_lib",  "file": "usr/lib/tachyon" },
+    { "type": "py_implib",   "file": "usr/lib/tachyon" }
+  ]
+}