Repository: spack/spack

Issue:
Installation issue: hip
### Steps to reproduce the issue

```console
$ spack spec -I hip
Input spec
--------------------------------
 -   hip

Concretized
--------------------------------
 -   hip@5.1.0%gcc@9.4.0~ipo build_type=Release patches=3b25db8 arch=linux-ubuntu20.04-zen2
[+]      ^cmake@3.23.1%gcc@9.4.0~doc+ncurses+ownlibs~qt build_type=Release arch=linux-ubuntu20.04-zen2
[+]          ^ncurses@6.2%gcc@9.4.0~symlinks+termlib abi=none arch=linux-ubuntu20.04-zen2
[+]              ^pkgconf@1.8.0%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]          ^openssl@1.1.1o%gcc@9.4.0~docs~shared certs=system arch=linux-ubuntu20.04-zen2
[+]              ^perl@5.34.1%gcc@9.4.0+cpanm+shared+threads arch=linux-ubuntu20.04-zen2
[+]                  ^berkeley-db@18.1.40%gcc@9.4.0+cxx~docs+stl patches=b231fcc arch=linux-ubuntu20.04-zen2
[+]                  ^bzip2@1.0.8%gcc@9.4.0~debug~pic+shared arch=linux-ubuntu20.04-zen2
[+]                      ^diffutils@3.8%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]                          ^libiconv@1.16%gcc@9.4.0 libs=shared,static arch=linux-ubuntu20.04-zen2
[+]                  ^gdbm@1.19%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]                      ^readline@8.1%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]                  ^zlib@1.2.12%gcc@9.4.0+optimize+pic+shared patches=0d38234 arch=linux-ubuntu20.04-zen2
[+]      ^comgr@5.1.0%gcc@9.4.0~ipo build_type=Release arch=linux-ubuntu20.04-zen2
[+]          ^llvm-amdgpu@5.1.0%gcc@9.4.0~ipo~link_llvm_dylib~llvm_dylib~openmp+rocm-device-libs build_type=Release arch=linux-ubuntu20.04-zen2
[+]              ^python@3.9.12%gcc@9.4.0+bz2+ctypes+dbm~debug+ensurepip+libxml2+lzma~nis~optimizations+pic+pyexpat+pythoncmd+readline+shared+sqlite3+ssl~tix~tkinter~ucs4+uuid+zlib patches=0d98e93,4c24573,f2fd060 arch=linux-ubuntu20.04-zen2
[+]                  ^expat@2.4.8%gcc@9.4.0+libbsd arch=linux-ubuntu20.04-zen2
[+]                      ^libbsd@0.11.5%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]                          ^libmd@1.0.4%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]                  ^gettext@0.21%gcc@9.4.0+bzip2+curses+git~libunistring+libxml2+tar+xz arch=linux-ubuntu20.04-zen2
[+]                      ^libxml2@2.9.13%gcc@9.4.0~python arch=linux-ubuntu20.04-zen2
[+]                          ^xz@5.2.5%gcc@9.4.0~pic libs=shared,static arch=linux-ubuntu20.04-zen2
[+]                      ^tar@1.34%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]                  ^libffi@3.4.2%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]                  ^sqlite@3.38.5%gcc@9.4.0+column_metadata+dynamic_extensions+fts~functions+rtree arch=linux-ubuntu20.04-zen2
[+]                  ^util-linux-uuid@2.37.4%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]              ^z3@4.8.16%gcc@9.4.0~gmp~ipo~python build_type=RelWithDebInfo arch=linux-ubuntu20.04-zen2
[+]          ^rocm-cmake@5.1.0%gcc@9.4.0~ipo build_type=Release arch=linux-ubuntu20.04-zen2
[+]      ^hsa-rocr-dev@5.1.0%gcc@9.4.0+image~ipo+shared build_type=Release patches=71e6851 arch=linux-ubuntu20.04-zen2
[+]          ^elfutils@0.186%gcc@9.4.0~bzip2~debuginfod+nls~xz arch=linux-ubuntu20.04-zen2
[+]              ^m4@1.4.19%gcc@9.4.0+sigsegv patches=9dc5fbd,bfdffa7 arch=linux-ubuntu20.04-zen2
[+]                  ^libsigsegv@2.13%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]          ^hsakmt-roct@5.1.0%gcc@9.4.0~ipo+shared build_type=Release patches=f926273 arch=linux-ubuntu20.04-zen2
[+]              ^libdrm@2.4.110%gcc@9.4.0~docs arch=linux-ubuntu20.04-zen2
[+]                  ^libpciaccess@0.16%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]                      ^libtool@2.4.7%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]                      ^util-macros@1.19.3%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]                  ^libpthread-stubs@0.4%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]                  ^meson@0.62.1%gcc@9.4.0 patches=aa6c50d arch=linux-ubuntu20.04-zen2
[+]                      ^ninja@1.10.2%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]                      ^py-pip@21.3.1%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]                      ^py-setuptools@59.4.0%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]                          ^py-wheel@0.37.0%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]              ^numactl@2.0.14%gcc@9.4.0 patches=4e1d78c,62fc8a8,ff37630 arch=linux-ubuntu20.04-zen2
[+]                  ^autoconf@2.69%gcc@9.4.0 patches=35c4492,7793209,a49dd5b arch=linux-ubuntu20.04-zen2
[+]                  ^automake@1.16.5%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]          ^xxd-standalone@8.2.1201%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
 -       ^mesa@22.0.2%gcc@9.4.0+glx+llvm+opengl~opengles+osmesa~strip buildtype=release default_library=shared patches=ada85be arch=linux-ubuntu20.04-zen2
[+]          ^bison@3.8.2%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]          ^flex@2.6.3%gcc@9.4.0+lex~nls arch=linux-ubuntu20.04-zen2
[+]              ^findutils@4.9.0%gcc@9.4.0 patches=440b954 arch=linux-ubuntu20.04-zen2
[+]          ^glproto@1.4.17%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]          ^libunwind@1.6.2%gcc@9.4.0~block_signals~conservative_checks~cxx_exceptions~debug~debug_frame+docs~pic+tests+weak_backtrace~xz~zlib components=none libs=shared,static arch=linux-ubuntu20.04-zen2
[+]          ^libx11@1.7.0%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]              ^inputproto@2.3.2%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]              ^kbproto@1.0.7%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]              ^libxcb@1.14%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]                  ^libxau@1.0.8%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]                      ^xproto@7.0.31%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]                  ^libxdmcp@1.1.2%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]                  ^xcb-proto@1.14.1%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]              ^xextproto@7.3.0%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]              ^xtrans@1.3.5%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]          ^libxext@1.3.3%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]          ^libxt@1.1.5%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]              ^libice@1.0.9%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]              ^libsm@1.2.3%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]          ^py-mako@1.1.6%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]              ^py-markupsafe@2.0.1%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]          ^xrandr@1.5.0%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]              ^libxrandr@1.5.0%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]                  ^libxrender@0.9.10%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]                      ^renderproto@0.11.1%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]                  ^randrproto@1.5.0%gcc@9.4.0 arch=linux-ubuntu20.04-zen2
[+]      ^rocminfo@5.1.0%gcc@9.4.0~ipo build_type=Release arch=linux-ubuntu20.04-zen2
[+]      ^roctracer-dev-api@5.1.0%gcc@9.4.0 arch=linux-ubuntu20.04-zen2

```


### Error message

<details><summary>Error message</summary><pre>
==> Installing mesa-22.0.2-dx3zb5p2ihdsjinct5p6nvjsnmnxttlv
==> No binary for mesa-22.0.2-dx3zb5p2ihdsjinct5p6nvjsnmnxttlv found: installing from source
==> Fetching https://mirror.spack.io/_source-cache/archive/df/df4fa560dcce6680133067cd15b0505fc424ca703244ce9ab247c74d2fab6885.tar.xz
==> Applied patch /root/spack/var/spack/repos/builtin/packages/mesa/mesa_check_llvm_version_suffix.patch
==> mesa: Executing phase: 'meson'
==> Error: ProcessError: Command exited with status 1:
    'meson' '/tmp/root/spack-stage/spack-stage-mesa-22.0.2-dx3zb5p2ihdsjinct5p6nvjsnmnxttlv/spack-src' '--prefix=/root/spack/opt/spack/linux-ubuntu20.04-zen2/gcc-9.4.0/mesa-22.0.2-dx3zb5p2ihdsjinct5p6nvjsnmnxttlv' '--libdir=/root/spack/opt/spack/linux-ubuntu20.04-zen2/gcc-9.4.0/mesa-22.0.2-dx3zb5p2ihdsjinct5p6nvjsnmnxttlv/lib' '-Dbuildtype=release' '-Dstrip=false' '-Ddefault_library=shared' '-Dvulkan-drivers=' '-Dgallium-vdpau=disabled' '-Dgallium-xvmc=disabled' '-Dgallium-omx=disabled' '-Dgallium-va=disabled' '-Dgallium-xa=disabled' '-Dgallium-nine=false' '-Dgallium-opencl=disabled' '-Dbuild-tests=false' '-Dglvnd=false' '-Dosmesa=true' '-Dglx=xlib' '-Degl=disabled' '-Dgbm=disabled' '-Ddri3=disabled' '-Dopengl=true' '-Dgles1=disabled' '-Dgles2=disabled' '-Dshared-glapi=enabled' '--native-file' 'meson-native-config.ini' '-Dllvm=enabled' '-Dshared-llvm=disabled' '-Dplatforms=x11' '-Dgallium-drivers=swrast' '-Ddri-drivers='

1 error found in build log:
     191    WARNING: CMake: Dependency m for LLVM was not found
     192    WARNING: CMake: Dependency rt for LLVM was not found
     193    WARNING: CMake: Dependency dl for LLVM was not found
     194    WARNING: CMake: Dependency m for LLVM was not found
     195    Run-time dependency LLVM (modules: bitwriter, core, coroutines, executionengine, instcombine, mcdisassembler, mcjit, native, scalaropts, transformutils) found: YES 14.0.0git
     196    
  >> 197    ../meson.build:1735:8: ERROR: Problem encountered: LLVM was built without RTTI, so Mesa must also disable RTTI. Use an LLVM built with LLVM_ENABLE_RTTI or add cpp_rtti=false.
     198    
     199    A full log can be found at /tmp/root/spack-stage/spack-stage-mesa-22.0.2-dx3zb5p2ihdsjinct5p6nvjsnmnxttlv/spack-src/spack-build/meson-logs/meson-log.txt

See build log for details:
  /tmp/root/spack-stage/spack-stage-mesa-22.0.2-dx3zb5p2ihdsjinct5p6nvjsnmnxttlv/spack-build-out.txt

==> Warning: Skipping build of hip-5.1.0-mbbjqg56sciut64ckhgopm4yjytewybm since mesa-22.0.2-dx3zb5p2ihdsjinct5p6nvjsnmnxttlv failed
</pre></details>


### Information on your system

* **Spack:** 0.18.0.dev0 (be45292a9c7139984c4929a35a7db19265543ff5)
* **Python:** 3.8.10
* **Platform:** linux-ubuntu20.04-zen2
* **Concretizer:** clingo


### Additional information

[spack-build-01-meson-out.txt](https://github.com/spack/spack/files/8666966/spack-build-01-meson-out.txt)
[spack-build-env.txt](https://github.com/spack/spack/files/8666967/spack-build-env.txt)
[spack-build-env-mods.txt](https://github.com/spack/spack/files/8666968/spack-build-env-mods.txt)
[spack-build-out.txt](https://github.com/spack/spack/files/8666969/spack-build-out.txt)

This is also a problem for `hip@5.0.0`, but I haven't tried testing earlier versions. This is similar to the problem fixed in #30102, but slightly different. As a workaround, you can specify `^mesa ~llvm`.

@arjun-raj-kuppala @haampie @srekolam

### General information

- [X] I have run `spack debug report` and reported the version of Spack/Python/Platform
- [X] I have run `spack maintainers <name-of-the-package>` and **@mentioned** any maintainers
- [X] I have uploaded the build log and environment files
- [X] I have searched the issues of this repo and believe this is not a duplicate


Discussion:
++ yes I've had to add `^mesa~llvm` to avoid this particular error condition.
@srekolam, @haampie, given that this issue has persisted for quite some time, perhaps we should just make hip itself say it depends on `mesa ~llvm`. 
https://github.com/spack/spack/pull/31066
@cgmb That was how it was, but it is incorrect. This should be constrained by fixing the `libllvm` versions required by mesa when `+llvm`. 

The correct fix for this is updating the `provides(libllvm)` versions for the new versions of `llvm-amdgpu` that were added to spack. Mesa already disables the llvm variant for incompatible versions of llvm.
@srekolam Looking at the build log, it seems `LLVM_ENABLE_RTTI` is required by mesa. Is this something that can be enabled in `llvm-amdgpu` and still be useful with HIP?
@kwryankrattiger  I see it is not enabled in the build scripts for the rocm releases. i can reconfirm one more time by talking to compiler team.
@kwryankrattiger  , Just had a chat with the compiler team and they confirm that it is not enabled for rocm releases. .
> @cgmb That was how it was, but it is incorrect. This should be constrained by fixing the `libllvm` versions required by mesa when `+llvm`.

`spack install hip` was broken since 7477161a05da0880aa642212df40f0dc910f26c9. After the change to depend on `gl` in cbfe0d749208285c54a7e9182baf4def62c00d84, the default install of hip has been broken for more time than it's been functional.

The fact that this problem went unaddressed for two months and Spack v0.18 got tagged with ROCm broken by default is incredibly disappointing. Mistakes happen, but this has dragged on for so long without correction I feel like there may be a deeper problem to fix than just the recipes themselves.
@cgmb  While I understand your frustration, as one of the people trying to maintain this stack I can assure you the known problems have not gone unaddressed. There have been various patches to stabilize the HIP install. Trust me when I tell you there is a lot of effort going in to making sure this works for everyone.

In an effort to better support ROCm, and other GPU, stacks in spack there is a plan in the works to add ROCm and CUDA CI pipelines to the spack CI. I am hopeful this will help alleviate some of these issues where HIP is quietly broken.
@srekolam Thanks for checking. I think maybe then there should be an `rtti` variant for `libllvm` providers so that mesa can check to make sure that feature is available when concertizing. `llvm-amdgpu` can just conflict when it is on and `mesa@22:` should toggle llvmpipe off in that case.

I did just try building it with RTTI enabled and it seems to almost work, it builds the libllvm at least.
> In an effort to better support ROCm, and other GPU, stacks in spack there is a plan in the works to add ROCm and CUDA CI pipelines to the spack CI. I am hopeful this will help alleviate some of these issues where HIP is quietly broken.

That sounds wonderful. I look forward to that day.
The ROCM stack takes so long to build, I'm afraid this would take days to execute the checks.
> The ROCM stack takes so long to build, I'm afraid this would take days to execute the checks.

@G-Ragghianti This has been considered and resources are being allocated appropriately to handle the additional load.

Although, the entire HIP stack should really only take about two hours to build, with LLVM being the biggest time sink. Where are you building that this is taking days?
I know math-libs (rocsolver ,rocblas, miopen-hip,rocfft etc ) do take good amount of time with rocblas taking multiple hours to complete. 
There are a number of ways to reduce the compile times without significantly reducing the value of those sorts of tests. For example, by choosing to build for a single amgpu_target, lowering the compiler optimization level, or building variants with fewer specialized kernels (e.g., `rocsolver ~optimal`).
It is good that the build time is getting some attention.  I didn't mean to imply that my rocm stack builds were taking days (probably more like 3 hours), but I have noticed that the spack CI checks have been taking longer to get queued in general.  
@cgmb It would be great if you could elaborate on these techniques to reduce compile time.  We try to do our own spack CI for our packages, but the compile of rocm limits the number of combinations that we can explore.

Resolving patch:
diff --git a/var/spack/repos/builtin/packages/llvm-amdgpu/package.py b/var/spack/repos/builtin/packages/llvm-amdgpu/package.py
--- a/var/spack/repos/builtin/packages/llvm-amdgpu/package.py
+++ b/var/spack/repos/builtin/packages/llvm-amdgpu/package.py
@@ -54,7 +54,8 @@ class LlvmAmdgpu(CMakePackage):
 
     provides('libllvm@11', when='@3.5:3.8')
     provides('libllvm@12', when='@3.9:4.2')
-    provides('libllvm@13', when='@4.3:')
+    provides('libllvm@13', when='@4.3:4.9')
+    provides('libllvm@14', when='@5:')
 
     depends_on('cmake@3.4.3:',  type='build', when='@:3.8')
     depends_on('cmake@3.13.4:', type='build', when='@3.9.0:')
diff --git a/var/spack/repos/builtin/packages/mesa/package.py b/var/spack/repos/builtin/packages/mesa/package.py
--- a/var/spack/repos/builtin/packages/mesa/package.py
+++ b/var/spack/repos/builtin/packages/mesa/package.py
@@ -204,6 +204,7 @@ def meson_args(self):
         args.append(opt_enable(num_frontends > 1, 'shared-glapi'))
 
         if '+llvm' in spec:
+            llvm_config = Executable(spec['libllvm'].prefix.bin.join('llvm-config'))
             # Fix builds on hosts where /usr/bin/llvm-config-* is found and provides an
             # incompatible version. Ensure that the llvm-config of spec['libllvm'] is
             # used.
@@ -212,12 +213,18 @@ def meson_args(self):
             mkdirp(self.build_directory)
             with working_dir(self.build_directory):
                 with open('meson-native-config.ini', 'w') as native_config:
-                    llvm_config = spec['libllvm'].prefix.bin + '/llvm-config'
                     native_config.write('[binaries]\n')
-                    native_config.write("llvm-config = '{0}'\n".format(llvm_config))
+                    native_config.write(
+                        "llvm-config = '{0}'\n".format(llvm_config.path))
             args.append('-Dllvm=enabled')
             args.append(opt_enable(
                 '+llvm_dylib' in spec['libllvm'], 'shared-llvm'))
+
+            # Match the llvm rtti setting
+            if llvm_config('--has-rtti', error=str, output=str).strip() == 'YES':
+                args.append('-Dcpp_rtti=true')
+            else:
+                args.append('-Dcpp_rtti=false')
         else:
             args.append('-Dllvm=disabled')
 


Test patch: