Repository: spack/spack

Issue:
Installation issue: py-tensorflow 2.11-2.12
### Steps to reproduce the issue

Tensorflow 2.11 and 2.12 build and can be imported in Python, but running a script that uses the GPU produces an error:

`Could not find compiler for platform CUDA: NOT_FOUND: could not find registered compiler for platform CUDA -- was support for that platform linked in?`

<details>
<summary>Script to reproduce</summary>

```python
# Adapted from: https://www.tensorflow.org/tutorials/quickstart/beginner

import tensorflow as tf
tf.get_logger().setLevel('DEBUG')
print("TensorFlow version:", tf.__version__)

mnist = tf.keras.datasets.mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0

model = tf.keras.models.Sequential([
  tf.keras.layers.Flatten(input_shape=(28, 28)),
  tf.keras.layers.Dense(128, activation='relu'),
  tf.keras.layers.Dropout(0.2),
  tf.keras.layers.Dense(10)
])
predictions = model(x_train[:1]).numpy()

loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)

model.compile(optimizer='adam',
              loss=loss_fn,
              metrics=['accuracy'])
model.fit(x_train, y_train, epochs=5)
model.evaluate(x_test,  y_test, verbose=2)

probability_model = tf.keras.Sequential([
  model,
  tf.keras.layers.Softmax()
])
```
</details>

<details>
<summary>Spec</summary>

```console
$ spack spec -I py-tensorflow@2.12 arch=linux-rocky8-broadwell +cuda cuda_arch==70 +nccl ^nccl+cuda cuda_arch==70
Input spec
--------------------------------
 -   py-tensorflow@2.12+cuda+nccl cuda_arch==70 arch=linux-rocky8-broadwell
 -       ^nccl+cuda cuda_arch==70

Concretized
--------------------------------
[+]  py-tensorflow@2.12.0%gcc@11.3.0~android~aws~computecpp+cuda~dynamic_kernels~gcp~gdr~hdfs~ignite~ios~jemalloc~kafka~mkl~monolithic~mpi+nccl~ngraph~numa~opencl~rocm~tensorrt~verbs~xla build_system=generic cuda_arch=70 patches=2017b3e arch=linux-rocky8-broadwell
[+]      ^bazel@5.3.0%gcc@11.3.0+nodepfail build_system=generic patches=0f6940d,3e6448a,53ca269,604423c,af73a49,dbbf38b arch=linux-rocky8-broadwell
[+]          ^openjdk@11.0.17_8%gcc@11.3.0 build_system=generic arch=linux-rocky8-broadwell
[+]          ^zip@3.0%gcc@11.3.0 build_system=makefile patches=14dc880,3bc30ba,5068e7c,51f48db,66ab4ce,a92fc4e,a95ed93,eb83fc8,f7d0bc4,fa8312c arch=linux-rocky8-broadwell
[+]      ^cuda@11.8.0%gcc@11.3.0~allow-unsupported-compilers~dev build_system=generic arch=linux-rocky8-broadwell
[+]          ^libxml2@2.10.3%gcc@11.3.0~python build_system=autotools arch=linux-rocky8-broadwell
[+]              ^libiconv@1.17%gcc@11.3.0 build_system=autotools libs=shared,static arch=linux-rocky8-broadwell
[+]      ^cudnn@8.4.0.27-11.6%gcc@11.3.0 build_system=generic arch=linux-rocky8-broadwell
[+]      ^hdf5@1.14.0%gcc@11.3.0~cxx~fortran+hl~ipo~java~map~mpi+shared~szip~threadsafe+tools api=default build_system=cmake build_type=RelWithDebInfo generator=make patches=0b5dd6f arch=linux-rocky8-broadwell
[+]          ^cmake@3.26.3%gcc@11.3.0~doc+ncurses+ownlibs~qt build_system=generic build_type=Release arch=linux-rocky8-broadwell
[+]          ^gmake@4.4.1%gcc@11.3.0~guile build_system=autotools arch=linux-rocky8-broadwell
[+]          ^pkgconf@1.8.0%gcc@11.3.0 build_system=autotools arch=linux-rocky8-broadwell
[+]          ^zlib@1.2.13%gcc@11.3.0+optimize+pic+shared build_system=makefile arch=linux-rocky8-broadwell
[+]      ^nccl@2.16.2-1%gcc@11.3.0+cuda build_system=makefile cuda_arch=70 arch=linux-rocky8-broadwell
[+]          ^rdma-core@41.0%gcc@11.3.0~ipo+static build_system=cmake build_type=RelWithDebInfo generator=make arch=linux-rocky8-broadwell
[+]              ^libnl@3.3.0%gcc@11.3.0 build_system=autotools arch=linux-rocky8-broadwell
[+]                  ^bison@3.8.2%gcc@11.3.0 build_system=autotools arch=linux-rocky8-broadwell
[+]                  ^flex@2.6.3%gcc@11.3.0+lex~nls build_system=autotools arch=linux-rocky8-broadwell
[+]                      ^findutils@4.9.0%gcc@11.3.0 build_system=autotools patches=440b954 arch=linux-rocky8-broadwell
[+]                  ^m4@1.4.19%gcc@11.3.0+sigsegv build_system=autotools patches=9dc5fbd,bfdffa7 arch=linux-rocky8-broadwell
[+]                      ^libsigsegv@2.14%gcc@11.3.0 build_system=autotools arch=linux-rocky8-broadwell
[+]              ^py-docutils@0.19%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]      ^protobuf@3.21.9%gcc@11.3.0~ipo+shared build_system=cmake build_type=Release generator=make arch=linux-rocky8-broadwell
[+]      ^py-absl-py@1.4.0%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]      ^py-astunparse@1.6.3%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]      ^py-flatbuffers@2.0.7%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]      ^py-gast@0.5.3%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]      ^py-google-pasta@0.2.0%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]      ^py-grpcio@1.52.0%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]          ^c-ares@1.15.0%gcc@11.3.0~ipo build_system=cmake build_type=RelWithDebInfo generator=make arch=linux-rocky8-broadwell
[+]          ^openssl@1.1.1t%gcc@11.3.0~docs~shared build_system=generic certs=mozilla arch=linux-rocky8-broadwell
[+]              ^ca-certificates-mozilla@2023-01-10%gcc@11.3.0 build_system=generic arch=linux-rocky8-broadwell
[+]              ^perl@5.36.0%gcc@11.3.0+cpanm+open+shared+threads build_system=generic arch=linux-rocky8-broadwell
[+]                  ^berkeley-db@18.1.40%gcc@11.3.0+cxx~docs+stl build_system=autotools patches=26090f4,b231fcc arch=linux-rocky8-broadwell
[+]          ^py-cython@0.29.33%gcc@11.3.0 build_system=python_pip patches=71de066 arch=linux-rocky8-broadwell
[+]          ^re2@2021-06-01%gcc@11.3.0~ipo+pic+shared build_system=cmake build_type=RelWithDebInfo generator=make arch=linux-rocky8-broadwell
[+]      ^py-h5py@3.8.0%gcc@11.3.0~mpi build_system=python_pip arch=linux-rocky8-broadwell
[+]          ^py-pkgconfig@1.5.5%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]              ^py-poetry-core@1.2.0%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]      ^py-jax@0.4.3%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]          ^py-jaxlib@0.4.3%gcc@11.3.0~cuda build_system=python_pip arch=linux-rocky8-broadwell
[+]          ^py-scipy@1.10.1%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]              ^py-build@0.10.0%gcc@11.3.0~virtualenv build_system=python_pip arch=linux-rocky8-broadwell
[+]                  ^py-pyproject-hooks@1.0.0%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]                  ^py-tomli@2.0.1%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]              ^py-meson-python@0.12.0%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]                  ^meson@1.1.0%gcc@11.3.0 build_system=python_pip patches=0f0b1bd arch=linux-rocky8-broadwell
[+]                  ^py-pyproject-metadata@0.7.1%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]              ^py-pybind11@2.10.1%gcc@11.3.0~ipo build_system=cmake build_type=RelWithDebInfo generator=ninja arch=linux-rocky8-broadwell
[+]              ^py-pythran@0.12.0%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]                  ^py-beniget@0.4.1%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]                  ^py-ply@3.11%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]      ^py-libclang@14.0.6%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]          ^llvm@14.0.6%gcc@11.3.0+clang+compiler-rt~cuda~flang+gold+internal_unwind~ipo+libcxx~link_llvm_dylib+lld+lldb+llvm_dylib~mlir+omp_as_runtime~omp_debug~omp_tsan+polly~python~split_dwarf~z3 build_system=cmake build_type=Release generator=ninja patches=6379168,d85ef51,f920173 shlib_symbol_version=none targets=none version_suffix=none arch=linux-rocky8-broadwell
[+]              ^binutils@2.40%gcc@11.3.0~gas+gold~gprofng~headers~interwork+ld~libiberty~lto~nls~pgo+plugins build_system=autotools libs=shared,static arch=linux-rocky8-broadwell
[+]                  ^texinfo@7.0%gcc@11.3.0 build_system=autotools arch=linux-rocky8-broadwell
[+]                  ^zstd@1.5.5%gcc@11.3.0+programs build_system=makefile compression=none libs=shared,static arch=linux-rocky8-broadwell
[+]              ^hwloc@2.9.1%gcc@11.3.0~cairo~cuda~gl~libudev+libxml2~netloc~nvml~oneapi-level-zero~opencl+pci~rocm build_system=autotools libs=shared,static arch=linux-rocky8-broadwell
[+]                  ^libpciaccess@0.17%gcc@11.3.0 build_system=autotools arch=linux-rocky8-broadwell
[+]                      ^libtool@2.4.7%gcc@11.3.0 build_system=autotools arch=linux-rocky8-broadwell
[+]                      ^util-macros@1.19.3%gcc@11.3.0 build_system=autotools arch=linux-rocky8-broadwell
[+]              ^libedit@3.1-20210216%gcc@11.3.0 build_system=autotools arch=linux-rocky8-broadwell
[+]              ^ninja@1.11.1%gcc@11.3.0+re2c build_system=generic arch=linux-rocky8-broadwell
[+]                  ^re2c@2.2%gcc@11.3.0 build_system=generic arch=linux-rocky8-broadwell
[+]              ^perl-data-dumper@2.173%gcc@11.3.0 build_system=perl arch=linux-rocky8-broadwell
[+]      ^py-numpy@1.23.5%gcc@11.3.0+blas+lapack build_system=python_pip patches=873745d arch=linux-rocky8-broadwell
[+]          ^openblas@0.3.23%gcc@11.3.0~bignuma~consistent_fpcsr+fortran~ilp64+locking+pic+shared build_system=makefile symbol_suffix=none threads=none arch=linux-rocky8-broadwell
[+]      ^py-opt-einsum@3.3.0%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]      ^py-packaging@23.0%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]          ^py-flit-core@3.7.1%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]      ^py-pip@23.0%gcc@11.3.0 build_system=generic arch=linux-rocky8-broadwell
[+]      ^py-protobuf@4.21.9%gcc@11.3.0+cpp build_system=python_pip arch=linux-rocky8-broadwell
[+]      ^py-requests@2.28.2%gcc@11.3.0~socks build_system=python_pip arch=linux-rocky8-broadwell
[+]          ^py-certifi@2022.12.7%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]          ^py-charset-normalizer@2.0.12%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]          ^py-idna@3.4%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]          ^py-urllib3@1.26.12%gcc@11.3.0~brotli~secure~socks build_system=python_pip arch=linux-rocky8-broadwell
[+]      ^py-setuptools@63.0.0%gcc@11.3.0 build_system=generic arch=linux-rocky8-broadwell
[+]      ^py-six@1.16.0%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]      ^py-tensorboard@2.12.0%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]          ^py-google-auth@2.16.2%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]              ^py-cachetools@5.2.0%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]              ^py-pyasn1-modules@0.2.8%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]                  ^py-pyasn1@0.4.8%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]              ^py-rsa@4.9%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]          ^py-google-auth-oauthlib@0.4.6%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]              ^py-requests-oauthlib@1.3.1%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]                  ^py-oauthlib@3.2.1%gcc@11.3.0~rsa~signals~signedtoken build_system=python_pip arch=linux-rocky8-broadwell
[+]          ^py-markdown@3.4.1%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]          ^py-tensorboard-data-server@0.7.0%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]              ^rust@1.65.0%gcc@11.3.0+analysis+clippy~rls+rustfmt+src build_system=generic extra_targets=none arch=linux-rocky8-broadwell
[+]                  ^curl@8.0.1%gcc@11.3.0~gssapi~ldap~libidn2~librtmp~libssh~libssh2+nghttp2 build_system=autotools libs=shared,static tls=openssl arch=linux-rocky8-broadwell
[+]                      ^nghttp2@1.52.0%gcc@11.3.0 build_system=autotools arch=linux-rocky8-broadwell
[+]                  ^libgit2@1.6.4%gcc@11.3.0~curl~ipo+mmap+ssh build_system=cmake build_type=RelWithDebInfo generator=make https=system arch=linux-rocky8-broadwell
[+]                      ^pcre@8.45%gcc@11.3.0~jit+multibyte+utf build_system=autotools arch=linux-rocky8-broadwell
[+]                  ^libssh2@1.10.0%gcc@11.3.0~ipo+shared build_system=cmake build_type=RelWithDebInfo crypto=openssl generator=make arch=linux-rocky8-broadwell
[+]          ^py-tensorboard-plugin-wit@1.8.1%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]          ^py-werkzeug@2.2.2%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]              ^py-markupsafe@2.1.1%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]      ^py-termcolor@1.1.0%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]      ^py-typing-extensions@4.5.0%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]      ^py-wheel@0.37.1%gcc@11.3.0 build_system=generic arch=linux-rocky8-broadwell
[+]      ^py-wrapt@1.14.1%gcc@11.3.0 build_system=python_pip arch=linux-rocky8-broadwell
[+]      ^python@3.10.10%gcc@11.3.0+bz2+crypt+ctypes+dbm~debug+libxml2+lzma~nis~optimizations+pic+pyexpat+pythoncmd+readline+shared+sqlite3+ssl~tkinter+uuid+zlib build_system=generic patches=0d98e93,7d40923,f2fd060 arch=linux-rocky8-broadwell
[+]          ^bzip2@1.0.8%gcc@11.3.0~debug~pic+shared build_system=generic arch=linux-rocky8-broadwell
[+]              ^diffutils@3.9%gcc@11.3.0 build_system=autotools arch=linux-rocky8-broadwell
[+]          ^expat@2.5.0%gcc@11.3.0+libbsd build_system=autotools arch=linux-rocky8-broadwell
[+]              ^libbsd@0.11.7%gcc@11.3.0 build_system=autotools arch=linux-rocky8-broadwell
[+]                  ^libmd@1.0.4%gcc@11.3.0 build_system=autotools arch=linux-rocky8-broadwell
[+]          ^gdbm@1.23%gcc@11.3.0 build_system=autotools arch=linux-rocky8-broadwell
[+]          ^gettext@0.21.1%gcc@11.3.0+bzip2+curses+git~libunistring+libxml2+tar+xz build_system=autotools arch=linux-rocky8-broadwell
[+]              ^tar@1.34%gcc@11.3.0 build_system=autotools zip=pigz arch=linux-rocky8-broadwell
[+]                  ^pigz@2.7%gcc@11.3.0 build_system=makefile arch=linux-rocky8-broadwell
[+]          ^libffi@3.4.4%gcc@11.3.0 build_system=autotools arch=linux-rocky8-broadwell
[+]          ^libxcrypt@4.4.33%gcc@11.3.0~obsolete_api build_system=autotools arch=linux-rocky8-broadwell
[+]          ^ncurses@6.4%gcc@11.3.0~symlinks+termlib abi=none build_system=autotools arch=linux-rocky8-broadwell
[+]          ^readline@8.2%gcc@11.3.0 build_system=autotools patches=bbf97f1 arch=linux-rocky8-broadwell
[+]          ^sqlite@3.40.1%gcc@11.3.0+column_metadata+dynamic_extensions+fts~functions+rtree build_system=autotools arch=linux-rocky8-broadwell
[+]          ^util-linux-uuid@2.38.1%gcc@11.3.0 build_system=autotools arch=linux-rocky8-broadwell
[+]          ^xz@5.4.1%gcc@11.3.0~pic build_system=autotools libs=shared,static arch=linux-rocky8-broadwell
[+]      ^swig@4.1.1%gcc@11.3.0 build_system=autotools arch=linux-rocky8-broadwell
[+]          ^pcre2@10.42%gcc@11.3.0~jit+multibyte build_system=autotools arch=linux-rocky8-broadwell
```
</details>

### Error message

The script runs successfully on the CPU but gives the following error on the GPU:

<details>
<summary>Error message</summary>

```console
$ python test_tensorflow.py
2023-05-05 17:08:51.342736: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX512F, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-05-05 17:09:06.986694: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1635] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 14633 MB memory:  -> device: 0, name: Tesla V100-PCIE-16GB, pci bus id: 0000:d8:00.0, compute capability: 7.0
TensorFlow version: 2.12.0
Epoch 1/5
2023-05-05 17:09:09.405093: W tensorflow/core/framework/op_kernel.cc:1830] OP_REQUIRES failed at xla_ops.cc:362 : UNIMPLEMENTED: Could not find compiler for platform CUDA: NOT_FOUND: could not find registered compiler for platform CUDA -- was support for that platform linked in?
2023-05-05 17:09:09.405175: I tensorflow/core/common_runtime/executor.cc:1197] [/job:localhost/replica:0/task:0/device:GPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): UNIMPLEMENTED: Could not find compiler for platform CUDA: NOT_FOUND: could not find registered compiler for platform CUDA -- was support for that platform linked in?
         [[{{node StatefulPartitionedCall_2}}]]
2023-05-05 17:09:09.405231: W tensorflow/core/framework/op_kernel.cc:1830] OP_REQUIRES failed at xla_ops.cc:362 : UNIMPLEMENTED: Could not find compiler for platform CUDA: NOT_FOUND: could not find registered compiler for platform CUDA -- was support for that platform linked in?
2023-05-05 17:09:09.405498: W tensorflow/core/framework/op_kernel.cc:1830] OP_REQUIRES failed at xla_ops.cc:362 : UNIMPLEMENTED: Could not find compiler for platform CUDA: NOT_FOUND: could not find registered compiler for platform CUDA -- was support for that platform linked in?
2023-05-05 17:09:09.405563: W tensorflow/core/framework/op_kernel.cc:1830] OP_REQUIRES failed at xla_ops.cc:362 : UNIMPLEMENTED: Could not find compiler for platform CUDA: NOT_FOUND: could not find registered compiler for platform CUDA -- was support for that platform linked in?
Traceback (most recent call last):
  File "/mnt/home/lgarrison/scc/repo/nixpack/test_tensorflow.py", line 24, in <module>
    model.fit(x_train, y_train, epochs=5)
  File "/mnt/home/lgarrison/scc/repo/spack/var/spack/environments/tfenv/.spack-env/view/lib/python3.10/site-packages/keras/utils/traceback_utils.py", line 70, in error_handler
    raise e.with_traceback(filtered_tb) from None
  File "/mnt/home/lgarrison/scc/repo/spack/var/spack/environments/tfenv/.spack-env/view/lib/python3.10/site-packages/tensorflow/python/eager/execute.py", line 52, in quick_execute
    tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
tensorflow.python.framework.errors_impl.UnimplementedError: Graph execution error:

Detected at node 'StatefulPartitionedCall_2' defined at (most recent call last):
    File "/mnt/home/lgarrison/scc/repo/nixpack/test_tensorflow.py", line 24, in <module>
      model.fit(x_train, y_train, epochs=5)
    File "/mnt/home/lgarrison/scc/repo/spack/var/spack/environments/tfenv/.spack-env/view/lib/python3.10/site-packages/keras/utils/traceback_utils.py", line 65, in error_handler
      return fn(*args, **kwargs)
    File "/mnt/home/lgarrison/scc/repo/spack/var/spack/environments/tfenv/.spack-env/view/lib/python3.10/site-packages/keras/engine/training.py", line 1685, in fit
      tmp_logs = self.train_function(iterator)
    File "/mnt/home/lgarrison/scc/repo/spack/var/spack/environments/tfenv/.spack-env/view/lib/python3.10/site-packages/keras/engine/training.py", line 1284, in train_function
      return step_function(self, iterator)
    File "/mnt/home/lgarrison/scc/repo/spack/var/spack/environments/tfenv/.spack-env/view/lib/python3.10/site-packages/keras/engine/training.py", line 1268, in step_function
      outputs = model.distribute_strategy.run(run_step, args=(data,))
    File "/mnt/home/lgarrison/scc/repo/spack/var/spack/environments/tfenv/.spack-env/view/lib/python3.10/site-packages/keras/engine/training.py", line 1249, in run_step
      outputs = model.train_step(data)
    File "/mnt/home/lgarrison/scc/repo/spack/var/spack/environments/tfenv/.spack-env/view/lib/python3.10/site-packages/keras/engine/training.py", line 1054, in train_step
      self.optimizer.minimize(loss, self.trainable_variables, tape=tape)
    File "/mnt/home/lgarrison/scc/repo/spack/var/spack/environments/tfenv/.spack-env/view/lib/python3.10/site-packages/keras/optimizers/optimizer.py", line 543, in minimize
      self.apply_gradients(grads_and_vars)
    File "/mnt/home/lgarrison/scc/repo/spack/var/spack/environments/tfenv/.spack-env/view/lib/python3.10/site-packages/keras/optimizers/optimizer.py", line 1174, in apply_gradients
      return super().apply_gradients(grads_and_vars, name=name)
    File "/mnt/home/lgarrison/scc/repo/spack/var/spack/environments/tfenv/.spack-env/view/lib/python3.10/site-packages/keras/optimizers/optimizer.py", line 650, in apply_gradients
      iteration = self._internal_apply_gradients(grads_and_vars)
    File "/mnt/home/lgarrison/scc/repo/spack/var/spack/environments/tfenv/.spack-env/view/lib/python3.10/site-packages/keras/optimizers/optimizer.py", line 1200, in _internal_apply_gradients
      return tf.__internal__.distribute.interim.maybe_merge_call(
    File "/mnt/home/lgarrison/scc/repo/spack/var/spack/environments/tfenv/.spack-env/view/lib/python3.10/site-packages/keras/optimizers/optimizer.py", line 1250, in _distributed_apply_gradients_fn
      distribution.extended.update(
    File "/mnt/home/lgarrison/scc/repo/spack/var/spack/environments/tfenv/.spack-env/view/lib/python3.10/site-packages/keras/optimizers/optimizer.py", line 1245, in apply_grad_to_update_var
      return self._update_step_xla(grad, var, id(self._var_key(var)))
Node: 'StatefulPartitionedCall_2'
Could not find compiler for platform CUDA: NOT_FOUND: could not find registered compiler for platform CUDA -- was support for that platform linked in?
         [[{{node StatefulPartitionedCall_2}}]] [Op:__inference_train_function_777]
srun: error: workergpu082: task 0: Exited with exit code 1
```
</details>

### Information on your system

Build system:

* **Spack:** 0.20.0.dev0 (d74b02f59a9722cc46652fe45c3a51a0e935c94e)
* **Python:** 3.6.8
* **Platform:** linux-rocky8-zen2
* **Concretizer:** clingo

The GPU test system is using a V100.

### Additional information

I've spent some time trying to figure out where this error is coming from, and my best guess (based on grepping the source for the error message) is that the NVPTX compiler registration isn't happening.  The registration is supposed to happen via static initialization here:

https://github.com/tensorflow/tensorflow/blob/v2.12.0/tensorflow/compiler/xla/service/gpu/nvptx_compiler_registration.cc

And if I use a Spack patch to put a `#error` compiler directive in that file, the build succeeds anyway.  So it looks like the compiler never sees that file, maybe indicating a problem in the build configuration or at the Bazel level.  I traced through the Bazel files a bit, but haven't been able to spot a problem.


[spack-build-env.txt](https://github.com/spack/spack/files/11410822/spack-build-env.txt)
[spack-build-out.txt.gz](https://github.com/spack/spack/files/11410824/spack-build-out.txt.gz) [15 MB file, 450 MB unzipped]


Maintainers:
@adamjstewart @aweits @pradyunsg

### General information

- [X] I have run `spack debug report` and reported the version of Spack/Python/Platform
- [X] I have run `spack maintainers <name-of-the-package>` and **@mentioned** any maintainers
- [X] I have uploaded the build log and environment files
- [X] I have searched the issues of this repo and believe this is not a duplicate


Discussion:
Build logs now attached.
a quick guess, can you try a build with +xla enabled? will look more when i get the chance...
That did it, thanks so much! I probably could/should have guessed that myself...

On the other hand, the script I was running was from the "quickstart" from the TensorFlow docs, so it seems like this is the default configuration that they expect TensorFlow to be built with. Do you think it would make sense to make `+xla` the default?
I would be in favor of matching the default settings from bazel. I changed PyTorch to match the default cmake settings. Would be nice to know why and when `~xla` fails so we can add a conflict though.
It looks like the defaults might be in https://github.com/tensorflow/tensorflow/blob/master/.bazelrc. Reading that file suggests the following changes to the current Spack defaults:
- `+xla` on Linux; disabled (maybe conflicts) on Android and iOS
- `+tensorrt` with CUDA on Linux*
- `+dynamic_kernels` on Linux

*It looks like `+tensorrt` by default might not be possible right now since TensorRT isn't in Spack.

It's hard to tell what version numbers all these defaults were introduced in. Git blame shows they disappear from some commits and reappear in others.  A conservative approach might be to just make these changes in `@2.11:`.

In terms of when `~xla` fails, in the example I ran, it seemed to need `+xla` when `+cuda`. But maybe there are other modes of using `+cuda` that don't require `+xla`, I don't know TensorFlow well enough to know.
> Reading that file suggests the following changes to the current Spack defaults

Can you submit a PR to make this change? I'll review and merge.

> A conservative approach might be to just make these changes in `@2.11:`

I would change the default for all versions. The correct way to disable the feature for old versions where the variant doesn't exist would be `when="@2.11:"`. If you want to use git or download and grep to find when these features were first introduced, that would be great. If not, I would leave the `when=...` off and wait for people to report bugs. I'm an "ask for forgiveness" person 😆 

> In terms of when `~xla` fails, in the example I ran, it seemed to need `+xla` when `+cuda`. But maybe there are other modes of using `+cuda` that don't require `+xla`, I don't know TensorFlow well enough to know.

I don't either unfortunately. Usually if this is the case, CMakeLists.txt will make one variant conditional on the other. Not sure if Bazel has an equivalent setting. You could report to the TF developers and see if they have an explanation.

Resolving patch:
diff --git a/var/spack/repos/builtin/packages/py-tensorflow/package.py b/var/spack/repos/builtin/packages/py-tensorflow/package.py
--- a/var/spack/repos/builtin/packages/py-tensorflow/package.py
+++ b/var/spack/repos/builtin/packages/py-tensorflow/package.py
@@ -129,13 +129,15 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension):
     variant("aws", default=False, description="Build with Amazon AWS Platform support")
     variant("kafka", default=False, description="Build with Apache Kafka Platform support")
     variant("ignite", default=False, description="Build with Apache Ignite support")
-    variant("xla", default=False, description="Build with XLA JIT support")
+    variant("xla", default=sys.platform != "darwin", description="Build with XLA JIT support")
     variant("gdr", default=False, description="Build with GDR support")
     variant("verbs", default=False, description="Build with libverbs support")
     variant("ngraph", default=False, description="Build with Intel nGraph support")
     variant("opencl", default=False, description="Build with OpenCL SYCL support")
     variant("computecpp", default=False, description="Build with ComputeCPP support")
-    variant("tensorrt", default=False, description="Build with TensorRT support")
+    variant(
+        "tensorrt", default=False, description="Build with TensorRT support"
+    )  # TODO: enable when TensorRT in Spack
     variant("cuda", default=sys.platform != "darwin", description="Build with CUDA support")
     variant(
         "nccl", default=sys.platform.startswith("linux"), description="Enable NVIDIA NCCL support"
@@ -146,7 +148,9 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension):
     variant("monolithic", default=False, description="Static monolithic build")
     variant("numa", default=False, description="Build with NUMA support")
     variant(
-        "dynamic_kernels", default=False, description="Build kernels into separate shared objects"
+        "dynamic_kernels",
+        default=sys.platform.startswith("linux"),
+        description="Build kernels into separate shared objects",
     )
 
     extends("python")


Test patch: