Repository: spack/spack

Issue:
Installation issue: py-matplotlib
### Steps to reproduce the issue

```console
$ spack spec -I <spec>
Input spec
--------------------------------
 -   py-matplotlib

Concretized
--------------------------------
 -   py-matplotlib@3.8.2%apple-clang@15.0.0~animation~fonts~latex~movies backend=macosx build_system=python_pip arch=darwin-ventura-m1
[+]      ^freetype@2.11.1%apple-clang@15.0.0+pic+shared build_system=autotools arch=darwin-ventura-m1
[+]          ^bzip2@1.0.8%apple-clang@15.0.0~debug~pic+shared build_system=generic arch=darwin-ventura-m1
[+]              ^diffutils@3.9%apple-clang@15.0.0 build_system=autotools arch=darwin-ventura-m1
[+]                  ^libiconv@1.17%apple-clang@15.0.0 build_system=autotools libs=shared,static arch=darwin-ventura-m1
[e]          ^gmake@3.81%apple-clang@15.0.0~guile build_system=generic patches=ca60bd9 arch=darwin-ventura-m1
[+]          ^gnuconfig@2022-09-17%apple-clang@15.0.0 build_system=generic arch=darwin-ventura-m1
[+]      ^libpng@1.6.39%apple-clang@15.0.0~ipo~pic build_system=cmake build_type=Release generator=make libs=shared,static arch=darwin-ventura-m1
[+]          ^cmake@3.27.9%apple-clang@15.0.0~doc+ncurses+ownlibs build_system=generic build_type=Release arch=darwin-ventura-m1
[+]              ^curl@8.4.0%apple-clang@15.0.0~gssapi~ldap~libidn2~librtmp~libssh~libssh2+nghttp2 build_system=autotools libs=shared,static tls=secure_transport arch=darwin-ventura-m1
[+]                  ^nghttp2@1.57.0%apple-clang@15.0.0 build_system=autotools arch=darwin-ventura-m1
[+]              ^ncurses@6.4%apple-clang@15.0.0~symlinks+termlib abi=none build_system=autotools arch=darwin-ventura-m1
[+]          ^zlib-ng@2.1.5%apple-clang@15.0.0+compat+opt build_system=autotools arch=darwin-ventura-m1
[+]      ^pkgconf@1.9.5%apple-clang@15.0.0 build_system=autotools arch=darwin-ventura-m1
[+]      ^py-certifi@2023.7.22%apple-clang@15.0.0 build_system=python_pip arch=darwin-ventura-m1
[+]      ^py-contourpy@1.0.7%apple-clang@15.0.0 build_system=python_pip arch=darwin-ventura-m1
[+]      ^py-cycler@0.11.0%apple-clang@15.0.0 build_system=python_pip arch=darwin-ventura-m1
[+]      ^py-fonttools@4.39.4%apple-clang@15.0.0 build_system=python_pip arch=darwin-ventura-m1
[+]      ^py-importlib-resources@5.12.0%apple-clang@15.0.0 build_system=python_pip arch=darwin-ventura-m1
[+]          ^py-zipp@3.17.0%apple-clang@15.0.0 build_system=python_pip arch=darwin-ventura-m1
[+]      ^py-kiwisolver@1.4.5%apple-clang@15.0.0 build_system=python_pip arch=darwin-ventura-m1
[+]          ^py-cppy@1.2.1%apple-clang@15.0.0 build_system=python_pip arch=darwin-ventura-m1
[+]      ^py-numpy@1.26.3%apple-clang@15.0.0 build_system=python_pip patches=873745d arch=darwin-ventura-m1
[+]          ^ninja@1.11.1%apple-clang@15.0.0+re2c build_system=generic arch=darwin-ventura-m1
[+]              ^re2c@2.2%apple-clang@15.0.0 build_system=generic arch=darwin-ventura-m1
[+]          ^openblas@0.3.26%apple-clang@15.0.0~bignuma~consistent_fpcsr+fortran~ilp64+locking+pic+shared build_system=makefile symbol_suffix=none threads=none arch=darwin-ventura-m1
[e]              ^perl@5.34.1%apple-clang@15.0.0~cpanm+opcode+open+shared+threads build_system=generic arch=darwin-ventura-m1
[+]          ^py-cython@3.0.8%apple-clang@15.0.0 build_system=python_pip arch=darwin-ventura-m1
[+]          ^py-pyproject-metadata@0.7.1%apple-clang@15.0.0 build_system=python_pip arch=darwin-ventura-m1
[+]          ^py-tomli@2.0.1%apple-clang@15.0.0 build_system=python_pip arch=darwin-ventura-m1
[+]      ^py-packaging@23.1%apple-clang@15.0.0 build_system=python_pip arch=darwin-ventura-m1
[+]          ^py-flit-core@3.9.0%apple-clang@15.0.0 build_system=python_pip arch=darwin-ventura-m1
[+]      ^py-pillow@10.0.0%apple-clang@15.0.0~freetype~imagequant+jpeg~jpeg2000~lcms~raqm~tiff~webp~xcb+zlib build_system=python_pip arch=darwin-ventura-m1
[+]          ^libjpeg-turbo@3.0.0%apple-clang@15.0.0~ipo~jpeg8~partial_decoder+shared+static build_system=cmake build_type=Release generator=make arch=darwin-ventura-m1
[+]              ^nasm@2.15.05%apple-clang@15.0.0 build_system=autotools arch=darwin-ventura-m1
[+]      ^py-pip@23.1.2%apple-clang@15.0.0 build_system=generic arch=darwin-ventura-m1
[+]      ^py-pybind11@2.11.1%apple-clang@15.0.0~ipo build_system=cmake build_type=Release generator=ninja arch=darwin-ventura-m1
[+]      ^py-pyparsing@3.0.9%apple-clang@15.0.0 build_system=python_pip arch=darwin-ventura-m1
[+]      ^py-python-dateutil@2.8.2%apple-clang@15.0.0 build_system=python_pip arch=darwin-ventura-m1
[+]          ^py-six@1.16.0%apple-clang@15.0.0 build_system=python_pip arch=darwin-ventura-m1
[+]      ^py-setuptools@68.0.0%apple-clang@15.0.0 build_system=generic arch=darwin-ventura-m1
[+]      ^py-setuptools-scm@7.1.0%apple-clang@15.0.0+toml build_system=python_pip arch=darwin-ventura-m1
[e]          ^git@2.39.3%apple-clang@15.0.0+man+nls+perl+subtree~svn~tcltk build_system=autotools arch=darwin-ventura-m1
[+]          ^py-typing-extensions@4.8.0%apple-clang@15.0.0 build_system=python_pip arch=darwin-ventura-m1
[+]      ^py-wheel@0.41.2%apple-clang@15.0.0 build_system=generic arch=darwin-ventura-m1
[e]      ^python@3.9.6%apple-clang@15.0.0+bz2+crypt+ctypes+dbm~debug+libxml2+lzma+nis~optimizations+pic+pyexpat~pythoncmd+readline+shared+sqlite3+ssl+tix+tkinter+uuid+zlib build_system=generic patches=0d98e93,ebdca64,f2fd060 arch=darwin-ventura-m1
[+]      ^qhull@2020.2%apple-clang@15.0.0~ipo build_system=cmake build_type=Release generator=make arch=darwin-ventura-m1
```


### Error message

<details><summary>raise NoHeadersError</summary>

<pre>
    raise NoHeadersError(msg.format(self.spec.name, include, platlib))
spack.error.NoHeadersError: Unable to locate py-numpy headers in /Users/srinath22/REPOS/Github/spack/opt/spack/darwin-ventura-m1/apple-clang-15.0.0/py-numpy-1.26.3-xrkzszwugmtpy4nanwppleujd2bkyigw/include/python3.9/numpy or /Users/srinath22/REPOS/Github/spack/opt/spack/darwin-ventura-m1/apple-clang-15.0.0/py-numpy-1.26.3-xrkzszwugmtpy4nanwppleujd2bkyigw/lib64/python3.9/site-packages/numpy
</pre></details>


### Information on your system

* **Spack:** 0.22.0.dev0 (5ef9bb775276c32e3a98b4f268a29d020e514c77)
* **Python:** 3.9.6
* **Platform:** darwin-ventura-m1
* **Concretizer:** clingo

### Additional information


@adamjstewart  Greg @becker33 pointed me to the platlib function in `python/package.py` calculating the incorrect location of py-numpy header files (some where near line 1094). We did an exercise of using `spack python` and it showed that the path calculated did not match py-numpy installed package. 

The stage directory is empy. So no `spack-build-*` files.   This error happens IN and OUT of an enviroment. 

### General information

- [X] I have run `spack debug report` and reported the version of Spack/Python/Platform
- [X] I have run `spack maintainers <name-of-the-package>` and **@mentioned** any maintainers
- [X] I have uploaded the build log and environment files
- [X] I have searched the issues of this repo and believe this is not a duplicate


Discussion:
What is the correct location for the headers?
On my laptop:
```
/Users/srinath22/REPOS/Github/spack/opt/spack/darwin-ventura-m1/apple-clang-15.0.0/py-numpy-1.26.3-xrkzszwugmtpy4nanwppleujd2bkyigw/lib/python3.9/site-packages/numpy/core/include/numpy/utils.h
```

maybe the _core_ is messing with things? But numpy appears twice.  So...
This could be a mattter where `py-numpy/package.py` is installing in an incorrect location.  I am not sure about python installation paths.

Can you try with this change and see if it works?
```diff
diff --git a/lib/spack/spack/build_systems/python.py b/lib/spack/spack/build_systems/python.py
index 39dc7519e8..67ede09da1 100644
--- a/lib/spack/spack/build_systems/python.py
+++ b/lib/spack/spack/build_systems/python.py
@@ -364,7 +364,7 @@ def libs(self) -> LibraryList:
         # Remove py- prefix in package name
         name = self.spec.name[3:]
 
-        root = self.prefix.join(self.spec["python"].package.platlib).join(name)
+        root = self.prefix.join(self.spec["python"].package.purelib).join(name)
 
         libs = fs.find_all_libraries(root, recursive=True)
 
```
Do I need to `spack concretize -f` again, or “make the change” and then do `spack install` ?

** Thank you for your time,

<>Srinath Vadlamani

From: Adam J. Stewart ***@***.***>
Date: Saturday, February 3, 2024 at 03:47
To: spack/spack ***@***.***>
Cc: Srinath Vadlamani ***@***.***>, Manual ***@***.***>
Subject: Re: [spack/spack] Installation issue: py-matplotlib (Issue #42441)

Can you try with this change and see if it works?

diff --git a/lib/spack/spack/build_systems/python.py b/lib/spack/spack/build_systems/python.py

index 39dc7519e8..67ede09da1 100644

--- a/lib/spack/spack/build_systems/python.py

+++ b/lib/spack/spack/build_systems/python.py

@@ -364,7 +364,7 @@

def libs(self) -> LibraryList:

# Remove py- prefix in package name

name = self.spec.name[3:]

-        root = self.prefix.join(self.spec["python"].package.platlib).join(name)

+        root = self.prefix.join(self.spec["python"].package.purelib).join(name)

libs = fs.find_all_libraries(root, recursive=True)

—
Reply to this email directly, [view it on GitHub](https://github.com/spack/spack/issues/42441#issuecomment-1925273621), or [unsubscribe](https://github.com/notifications/unsubscribe-auth/AAEO6CPE7TXT5M7CGWKWUSLYRYITJAVCNFSM6AAAAABCV4BQ76VHI2DSMVQWIX3LMV43OSLTON2WKQ3PNVWWK3TUHMYTSMRVGI3TGNRSGE).
You are receiving this because you are subscribed to this thread.[Image removed by sender.]Message ID: ***@***.***>
Probably. Would be easier to debug if you can reproduce outside an environment.
Out of an env:
```
diff --git a/lib/spack/spack/build_systems/python.py b/lib/spack/spack/build_systems/python.py
index 39dc7519..1aa6cf8c 100644
--- a/lib/spack/spack/build_systems/python.py
+++ b/lib/spack/spack/build_systems/python.py
@@ -348,7 +348,7 @@ def headers(self) -> HeaderList:

         # Headers may be in either location
         include = self.prefix.join(self.spec["python"].package.include).join(name)
-        platlib = self.prefix.join(self.spec["python"].package.platlib).join(name)
+        platlib = self.prefix.join(self.spec["python"].package.purelib).join(name)
         headers = fs.find_all_headers(include) + fs.find_all_headers(platlib)

         if headers:
@@ -364,7 +364,7 @@ def libs(self) -> LibraryList:
         # Remove py- prefix in package name
         name = self.spec.name[3:]

-        root = self.prefix.join(self.spec["python"].package.platlib).join(name)
+        root = self.prefix.join(self.spec["python"].package.purelib).join(name)

         libs = fs.find_all_libraries(root, recursive=True)

(END)
```
Fixed my issue.  Should I submit a PR or can you apply? 



@rgommers what do you think? I was under the impression that numpy (and the libs/headers of other Python libraries) would be installed to platlib, not purelib. Does it depend on the build backend? If we need to, we can always check both platlib and purelib, that might be the most robust solution.
@adamjstewart, I like the "rhobust" solution.  Check with both facitilities.  
Checking if there is more I should do here.
Nope, just waiting for @rgommers's opinion before I submit a PR.
@srinathv what is the result of ```spack config get compilers```?
Out side of my enviroment:
```
% spack config get compilers
compilers:
- compiler:
    spec: apple-clang@=15.0.0
    paths:
      cc: /usr/bin/clang
      cxx: /usr/bin/clang++
      f77: null
      fc: null
    flags: {}
    operating_system: ventura
    target: aarch64
    modules: []
    environment: {}
    extra_rpaths: []
- compiler:
    spec: gcc@=12.2.0
    paths:
      cc: null
      cxx: null
      f77: /usr/local/bin/gfortran
      fc: /usr/local/bin/gfortran
    flags: {}
    operating_system: ventura
    target: aarch64
    modules: []
    environment: {}
    extra_rpaths: []
    ```
    
    
    
Let it be known that I have moved to using `pip3` to manage my python packages on my local mac os laptop.  So my attention to the python packages in spack is reduced.
> @rgommers what do you think? I was under the impression that numpy (and the libs/headers of other Python libraries) would be installed to platlib, not purelib. Does it depend on the build backend? If we need to, we can always check both platlib and purelib, that might be the most robust solution.

It should always be platlib; purelib cannot be right. In almost all distros those point to the same place though - making them different is fairly useful and opens you up to subtle bugs (I'm fairly sure this is not a NumPy bug though). 

Checking both platlib and purelib seems to solve it and probably can't hurt, so if the actual bug cannot be located it's not a problem I think to do this.

> maybe the _core_ is messing with things? But numpy appears twice.  So...

That path looks fine to me. numpy headers go into `/path/to/site-packages/numpy/core/include/numpy/`. They're used with `#include "numpy/hdrname.h"`, so `numpy` appearing twice is correct.

Resolving patch:
diff --git a/lib/spack/spack/build_systems/python.py b/lib/spack/spack/build_systems/python.py
--- a/lib/spack/spack/build_systems/python.py
+++ b/lib/spack/spack/build_systems/python.py
@@ -2,7 +2,10 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import functools
 import inspect
+import operator
 import os
 import re
 import shutil
@@ -346,16 +349,20 @@ def headers(self) -> HeaderList:
         # Remove py- prefix in package name
         name = self.spec.name[3:]
 
-        # Headers may be in either location
+        # Headers should only be in include or platlib, but no harm in checking purelib too
         include = self.prefix.join(self.spec["python"].package.include).join(name)
         platlib = self.prefix.join(self.spec["python"].package.platlib).join(name)
-        headers = fs.find_all_headers(include) + fs.find_all_headers(platlib)
+        purelib = self.prefix.join(self.spec["python"].package.purelib).join(name)
+
+        find_all_headers = functools.partial(fs.find_all_headers, recursive=True)
+        headers_list = map(find_all_headers, [include, platlib, purelib])
+        headers = functools.reduce(operator.add, headers_list)
 
         if headers:
             return headers
 
-        msg = "Unable to locate {} headers in {} or {}"
-        raise NoHeadersError(msg.format(self.spec.name, include, platlib))
+        msg = "Unable to locate {} headers in {}, {}, or {}"
+        raise NoHeadersError(msg.format(self.spec.name, include, platlib, purelib))
 
     @property
     def libs(self) -> LibraryList:
@@ -364,15 +371,19 @@ def libs(self) -> LibraryList:
         # Remove py- prefix in package name
         name = self.spec.name[3:]
 
-        root = self.prefix.join(self.spec["python"].package.platlib).join(name)
+        # Libraries should only be in platlib, but no harm in checking purelib too
+        platlib = self.prefix.join(self.spec["python"].package.platlib).join(name)
+        purelib = self.prefix.join(self.spec["python"].package.purelib).join(name)
 
-        libs = fs.find_all_libraries(root, recursive=True)
+        find_all_libraries = functools.partial(fs.find_all_libraries, recursive=True)
+        libs_list = map(find_all_libraries, [platlib, purelib])
+        libs = functools.reduce(operator.add, libs_list)
 
         if libs:
             return libs
 
-        msg = "Unable to recursively locate {} libraries in {}"
-        raise NoLibrariesError(msg.format(self.spec.name, root))
+        msg = "Unable to recursively locate {} libraries in {} or {}"
+        raise NoLibrariesError(msg.format(self.spec.name, platlib, purelib))
 
 
 @spack.builder.builder("python_pip")


Test patch: