Change:
, linuxPackages , darwin <del> , enableRocm ? false , enableCuda ? false </del> <ins> # one of `[ null "rocm" "cuda" ]` , acceleration ? null </ins> }: let pname = "ollama"; <del> version = "0.1.24"; </del> <ins> version = "0.1.26"; </ins> <del> warnIfNotLinux = warning: (lib.warnIfNot stdenv.isLinux warning stdenv.isLinux); gpuWarning = api: "building ollama with ${api} is only supported on linux; falling back to cpu"; rocmIsEnabled = enableRocm && (warnIfNotLinux (gpuWarning "rocm")); cudaIsEnabled = enableCuda && (warnIfNotLinux (gpuWarning "cuda")); enableLinuxGpu = rocmIsEnabled || cudaIsEnabled; appleFrameworks = darwin.apple_sdk_11_0.frameworks; metalFrameworks = [ appleFrameworks.Accelerate appleFrameworks.Metal appleFrameworks.MetalKit appleFrameworks.MetalPerformanceShaders ]; src = fetchFromGitHub { owner = "jmorganca"; repo = "ollama"; rev = "v${version}"; hash = "sha256-GwZA1QUH8I8m2bGToIcMMaB5MBnioQP4+n1SauUJYP8="; fetchSubmodules = true; }; preparePatch = patch: hash: fetchpatch { url = "file://${src}/llm/patches/${patch}"; inherit hash; stripLen = 1; extraPrefix = "llm/llama.cpp/"; }; inherit (lib) licenses platforms maintainers; ollama = { inherit pname version src; vendorHash = "sha256-wXRbfnkbeXPTOalm7SFLvHQ9j46S/yLNbFy+OWNSamQ="; nativeBuildInputs = [ cmake ] ++ lib.optionals enableLinuxGpu [ makeWrapper ] ++ lib.optionals stdenv.isDarwin metalFrameworks; patches = [ # remove uses of `git` in the `go generate` script # instead use `patch` where necessary ./remove-git.patch # replace a hardcoded use of `g++` with `$CXX` ./replace-gcc.patch # ollama's patches of llama.cpp's example server # `ollama/llm/generate/gen_common.sh` -> "apply temporary patches until fix is upstream" (preparePatch "01-cache.diff" "sha256-PC4yN98hFvK+PEITiDihL8ki3bJuLVXrAm0CGf8GPJE=") (preparePatch "02-shutdown.diff" "sha256-cElAp9Z9exxN964vB/YFuBhZoEcoAwGSMCnbh+l/V4Q=") ]; postPatch = '' # use a patch from the nix store in the `go generate` script substituteInPlace llm/generate/gen_common.sh  --subst-var-by cmakeIncludePatch '${./cmake-include.patch}' # `ollama/llm/generate/gen_common.sh` -> "avoid duplicate main symbols when we link into the cgo binary" substituteInPlace llm/llama.cpp/examples/server/server.cpp  --replace-fail 'int main(' 'int __main(' # replace inaccurate version number with actual release version substituteInPlace version/version.go --replace-fail 0.0.0 '${version}' ''; preBuild = '' export OLLAMA_SKIP_PATCHING=true # build llama.cpp libraries for ollama go generate ./... ''; ldflags = [ "-s" "-w" "-X=github.com/jmorganca/ollama/version.Version=${version}" "-X=github.com/jmorganca/ollama/server.mode=release" ]; meta = { description = "Get up and running with large language models locally"; homepage = "https://github.com/jmorganca/ollama"; license = licenses.mit; platforms = platforms.unix; mainProgram = "ollama"; maintainers = with maintainers; [ abysssol dit7ya elohmeier ]; }; }; </del> <ins> validAccel = lib.assertOneOf "ollama.acceleration" acceleration [ null "rocm" "cuda" ]; warnIfNotLinux = api: (lib.warnIfNot stdenv.isLinux "building ollama with `${api}` is only supported on linux; falling back to cpu" stdenv.isLinux); enableRocm = validAccel && (acceleration == "rocm") && (warnIfNotLinux "rocm"); enableCuda = validAccel && (acceleration == "cuda") && (warnIfNotLinux "cuda"); </ins> rocmClang = linkFarm "rocm-clang" { llvm = rocmPackages.llvm.clang;

Discussion:
Ollama can't discover GPU libraries Steps to reproduce the behavior: this completely not overkill nixos config: Ollama discover GPU libraries and able to use GPU acceleration. Previously I was able to discover GPU libs by running downloaded ollama binary via - But anyway it crushes because it uses ROCM 6.0 while in nixpkgs unstable we still have 5.7.1 <!-- Please @ people who are in the list of the offending package or module. If in doubt, check for whoever last touched something. -- Please run and paste the result. Add a :+1: [reaction] to [issues you find important]. [reaction]: [issues you find important]:
This appears to be happening because the ollama service currently only ever builds the ollama package in cpu mode. If my understanding is correct, then should fix this, though gpu support won't be enabled automatically. Once is merged, rocm would be enabled like this, and should work without any other configuration: