Change:
Ollama can't discover GPU libraries Steps to reproduce the behavior: this completely not overkill nixos config: Ollama discover GPU libraries and able to use GPU acceleration. Previously I was able to discover GPU libs by running downloaded ollama binary via - But anyway it crushes because it uses ROCM 6.0 while in nixpkgs unstable we still have 5.7.1 <!-- Please @ people who are in the list of the offending package or module. If in doubt, check for whoever last touched something. -- Please run and paste the result. Add a :+1: [reaction] to [issues you find important]. [reaction]: [issues you find important]:
This appears to be happening because the ollama service currently only ever builds the ollama package in cpu mode. If my understanding is correct, then should fix this, though gpu support won't be enabled automatically. Once is merged, rocm would be enabled like this, and should work without any other configuration:

Discussion:
<del> { config, lib, pkgs, ... }: let </del> <ins> { config, lib, pkgs, ... }: let inherit (lib.types) nullOr enum; </ins> cfg = config.services.ollama; <del> in { </del> <ins> ollamaPackage = cfg.package.override { inherit (cfg) acceleration; linuxPackages.nvidia_x11 = config.hardware.nvidia.package; }; in { </ins> options = { services.ollama = { enable = lib.mkEnableOption (

Specifies the bind address on which the ollama server HTTP interface listens. ''; }; <ins> acceleration = lib.mkOption { type = nullOr (enum [ "rocm" "cuda" ]); default = null; example = "rocm"; description = lib.mdDoc '' Specifies the interface to use for hardware acceleration. - `rocm`: supported by modern AMD GPUs - `cuda`: supported by modern NVIDIA GPUs ''; }; </ins> package = lib.mkPackageOption pkgs "ollama" { }; }; }; config = lib.mkIf cfg.enable { systemd = { services.ollama = { wantedBy = [ "multi-user.target" ];

OLLAMA_HOST = cfg.listenAddress; }; serviceConfig = { <del> ExecStart = "${lib.getExe cfg.package} serve"; </del> <ins> ExecStart = "${lib.getExe ollamaPackage} serve"; </ins> WorkingDirectory = "/var/lib/ollama"; StateDirectory = [ "ollama" ]; DynamicUser = true;

}; }; <del> environment.systemPackages = [ cfg.package ]; </del> <ins> environment.systemPackages = [ ollamaPackage ]; </ins> }; <del> meta.maintainers = with lib.maintainers; [ onny ]; </del> <ins> meta.maintainers = with lib.maintainers; [ abysssol onny ]; </ins> }

, makeWrapper , stdenv <ins> , pkgs </ins> , cmake , gcc12 , clblast

, linuxPackages , darwin <del> , enableRocm ? false , enableCuda ? false </del> <ins> # one of `[ null "rocm" "cuda" ]` , acceleration ? null </ins> }: let pname = "ollama"; <del> version = "0.1.24"; </del> <ins> version = "0.1.26"; </ins> <del> warnIfNotLinux = warning: (lib.warnIfNot stdenv.isLinux warning stdenv.isLinux); gpuWarning = api: "building ollama with ${api} is only supported on linux; falling back to cpu"; rocmIsEnabled = enableRocm && (warnIfNotLinux (gpuWarning "rocm")); cudaIsEnabled = enableCuda && (warnIfNotLinux (gpuWarning "cuda")); enableLinuxGpu = rocmIsEnabled || cudaIsEnabled; appleFrameworks = darwin.apple_sdk_11_0.frameworks; metalFrameworks = [ appleFrameworks.Accelerate appleFrameworks.Metal appleFrameworks.MetalKit appleFrameworks.MetalPerformanceShaders ]; src = fetchFromGitHub { owner = "jmorganca"; repo = "ollama"; rev = "v${version}"; hash = "sha256-GwZA1QUH8I8m2bGToIcMMaB5MBnioQP4+n1SauUJYP8="; fetchSubmodules = true; }; preparePatch = patch: hash: fetchpatch { url = "file://${src}/llm/patches/${patch}"; inherit hash; stripLen = 1; extraPrefix = "llm/llama.cpp/"; }; inherit (lib) licenses platforms maintainers; ollama = { inherit pname version src; vendorHash = "sha256-wXRbfnkbeXPTOalm7SFLvHQ9j46S/yLNbFy+OWNSamQ="; nativeBuildInputs = [ cmake ] ++ lib.optionals enableLinuxGpu [ makeWrapper ] ++ lib.optionals stdenv.isDarwin metalFrameworks; patches = [ # remove uses of `git` in the `go generate` script # instead use `patch` where necessary ./remove-git.patch # replace a hardcoded use of `g++` with `$CXX` ./replace-gcc.patch # ollama's patches of llama.cpp's example server # `ollama/llm/generate/gen_common.sh` -> "apply temporary patches until fix is upstream" (preparePatch "01-cache.diff" "sha256-PC4yN98hFvK+PEITiDihL8ki3bJuLVXrAm0CGf8GPJE=") (preparePatch "02-shutdown.diff" "sha256-cElAp9Z9exxN964vB/YFuBhZoEcoAwGSMCnbh+l/V4Q=") ]; postPatch = '' # use a patch from the nix store in the `go generate` script substituteInPlace llm/generate/gen_common.sh  --subst-var-by cmakeIncludePatch '${./cmake-include.patch}' # `ollama/llm/generate/gen_common.sh` -> "avoid duplicate main symbols when we link into the cgo binary" substituteInPlace llm/llama.cpp/examples/server/server.cpp  --replace-fail 'int main(' 'int __main(' # replace inaccurate version number with actual release version substituteInPlace version/version.go --replace-fail 0.0.0 '${version}' ''; preBuild = '' export OLLAMA_SKIP_PATCHING=true # build llama.cpp libraries for ollama go generate ./... ''; ldflags = [ "-s" "-w" "-X=github.com/jmorganca/ollama/version.Version=${version}" "-X=github.com/jmorganca/ollama/server.mode=release" ]; meta = { description = "Get up and running with large language models locally"; homepage = "https://github.com/jmorganca/ollama"; license = licenses.mit; platforms = platforms.unix; mainProgram = "ollama"; maintainers = with maintainers; [ abysssol dit7ya elohmeier ]; }; }; </del> <ins> validAccel = lib.assertOneOf "ollama.acceleration" acceleration [ null "rocm" "cuda" ]; warnIfNotLinux = api: (lib.warnIfNot stdenv.isLinux "building ollama with `${api}` is only supported on linux; falling back to cpu" stdenv.isLinux); enableRocm = validAccel && (acceleration == "rocm") && (warnIfNotLinux "rocm"); enableCuda = validAccel && (acceleration == "cuda") && (warnIfNotLinux "cuda"); </ins> rocmClang = linkFarm "rocm-clang" { llvm = rocmPackages.llvm.clang;

rocmClang ]; }; <del> rocmVars = { ROCM_PATH = rocmPath; CLBlast_DIR = "${clblast}/lib/cmake/CLBlast"; }; </del> cudaToolkit = buildEnv { name = "cuda-toolkit";

cudaPackages.cuda_cudart ]; }; <del> cudaVars = { CUDA_LIB_DIR = "${cudaToolkit}/lib"; CUDACXX = "${cudaToolkit}/bin/nvcc"; CUDAToolkit_ROOT = cudaToolkit; }; linuxGpuLibs = { buildInputs = lib.optionals rocmIsEnabled [ rocmPackages.clr rocmPackages.hipblas rocmPackages.rocblas rocmPackages.rocsolver rocmPackages.rocsparse libdrm ] ++ lib.optionals cudaIsEnabled [ cudaPackages.cuda_cudart ]; }; appleGpuLibs = { buildInputs = metalFrameworks; }; runtimeLibs = lib.optionals rocmIsEnabled [ </del> <ins> runtimeLibs = lib.optionals enableRocm [ </ins> rocmPackages.rocm-smi <del> ] ++ lib.optionals cudaIsEnabled [ </del> <ins> ] ++ lib.optionals enableCuda [ </ins> linuxPackages.nvidia_x11 ]; <del> runtimeLibWrapper = { postFixup = '' mv "$out/bin/${pname}" "$out/bin/.${pname}-unwrapped" makeWrapper "$out/bin/.${pname}-unwrapped" "$out/bin/${pname}"  --suffix LD_LIBRARY_PATH : '${lib.makeLibraryPath runtimeLibs}' ''; }; </del> <ins> appleFrameworks = darwin.apple_sdk_11_0.frameworks; metalFrameworks = [ appleFrameworks.Accelerate appleFrameworks.Metal appleFrameworks.MetalKit appleFrameworks.MetalPerformanceShaders ]; </ins> goBuild = <del> if cudaIsEnabled then </del> <ins> if enableCuda then </ins> buildGoModule.override { stdenv = overrideCC stdenv gcc12; } else buildGoModule; <ins> src = fetchFromGitHub { owner = "jmorganca"; repo = "ollama"; rev = "v${version}"; hash = "sha256-Kw3tt9ayEMgI2V6OeaOkWfNwqfCL7MDD/nN5iXk5LnY="; fetchSubmodules = true; }; preparePatch = patch: hash: fetchpatch { url = "file://${src}/llm/patches/${patch}"; inherit hash; stripLen = 1; extraPrefix = "llm/llama.cpp/"; }; inherit (lib) licenses platforms maintainers; </ins> in <del> goBuild (ollama // (lib.optionalAttrs rocmIsEnabled rocmVars) // (lib.optionalAttrs cudaIsEnabled cudaVars) // (lib.optionalAttrs enableLinuxGpu linuxGpuLibs) // (lib.optionalAttrs enableLinuxGpu runtimeLibWrapper) </del> <ins> goBuild ((lib.optionalAttrs enableRocm { ROCM_PATH = rocmPath; CLBlast_DIR = "${clblast}/lib/cmake/CLBlast"; }) // (lib.optionalAttrs enableCuda { CUDA_LIB_DIR = "${cudaToolkit}/lib"; CUDACXX = "${cudaToolkit}/bin/nvcc"; CUDAToolkit_ROOT = cudaToolkit; }) // { inherit pname version src; vendorHash = "sha256-zTrBighPBqZ9hhkEV3UawJZUYyPRay7+P6wkhDtpY7M="; nativeBuildInputs = [ cmake ] ++ lib.optionals (enableRocm || enableCuda) [ makeWrapper ] ++ lib.optionals stdenv.isDarwin metalFrameworks; buildInputs = lib.optionals enableRocm [ rocmPackages.clr rocmPackages.hipblas rocmPackages.rocblas rocmPackages.rocsolver rocmPackages.rocsparse libdrm ] ++ lib.optionals enableCuda [ cudaPackages.cuda_cudart ] ++ lib.optionals stdenv.isDarwin metalFrameworks; patches = [ # remove uses of `git` in the `go generate` script # instead use `patch` where necessary ./remove-git.patch # replace a hardcoded use of `g++` with `$CXX` ./replace-gcc.patch # ollama's patches of llama.cpp's example server # `ollama/llm/generate/gen_common.sh` -> "apply temporary patches until fix is upstream" (preparePatch "01-cache.diff" "sha256-MTTln2G0G8dntihUzEjPM1ruTsApb4ZToBczJb8EG68=") (preparePatch "02-cudaleaks.diff" "sha256-Cu7E9iEcvddPL9mPPI5Z96qmwWigi3f0WgSpPRjGc88=") ]; postPatch = '' # use a patch from the nix store in the `go generate` script substituteInPlace llm/generate/gen_common.sh  --subst-var-by cmakeIncludePatch '${./cmake-include.patch}' # `ollama/llm/generate/gen_common.sh` -> "avoid duplicate main symbols when we link into the cgo binary" substituteInPlace llm/llama.cpp/examples/server/server.cpp  --replace-fail 'int main(' 'int __main(' # replace inaccurate version number with actual release version substituteInPlace version/version.go --replace-fail 0.0.0 '${version}' ''; preBuild = '' export OLLAMA_SKIP_PATCHING=true # build llama.cpp libraries for ollama go generate ./... ''; postFixup = '' # the app doesn't appear functional at the moment, so hide it mv "$out/bin/app" "$out/bin/.ollama-app" '' + lib.optionalString (enableRocm || enableCuda) '' # expose runtime libraries necessary to use the gpu mv "$out/bin/ollama" "$out/bin/.ollama-unwrapped" makeWrapper "$out/bin/.ollama-unwrapped" "$out/bin/ollama"  --suffix LD_LIBRARY_PATH : '/run/opengl-driver/lib:${lib.makeLibraryPath runtimeLibs}' ''; ldflags = [ "-s" "-w" "-X=github.com/jmorganca/ollama/version.Version=${version}" "-X=github.com/jmorganca/ollama/server.mode=release" ]; </ins> <del> // (lib.optionalAttrs stdenv.isDarwin appleGpuLibs)) </del> <ins> # for now, just test that rocm and cuda build passthru.tests = lib.optionalAttrs stdenv.isLinux { rocm = pkgs.ollama.override { acceleration = "rocm"; }; cuda = pkgs.ollama.override { acceleration = "cuda"; }; }; meta = { description = "Get up and running with large language models locally"; homepage = "https://github.com/jmorganca/ollama"; license = licenses.mit; platforms = platforms.unix; mainProgram = "ollama"; maintainers = with maintainers; [ abysssol dit7ya elohmeier ]; }; }) </ins>