Change:
Running fails when launching the runner: Steps to reproduce the behavior: with the server a model and observe that it fails Either should contain , or the wrapper should add wherever is to . When starting, also prints this warning: . I have 2x Quadro P5000s, using the closed drivers since open doesn't support them. <!-- Please @ people who are in the list of the offending package or module. If in doubt, check for whoever last touched something. --Please run and paste the result. Add a :+1: [reaction] to [issues you find important]. [reaction]: [issues you find important]:
I'm not actually a maintainer of ollama, but there are three others who should have received the ping. That said, I wonder if this is fixed by
Sorry for the ping, I saw your PR and thought the same thing. I've tried with and without the changes from that PR and it doesn't fix the issue.
This awful hack fixes the issue for me:
I wonder if my change could be resolved by putting in LDLIBRARYPATH. I'll try and do some testing in the next couple days

Discussion:
paths = rocmLibs ++ [ rocmClang ]; }; <ins> cudaLibs = [ cudaPackages.cuda_cudart cudaPackages.libcublas cudaPackages.cuda_cccl ]; </ins> cudaToolkit = buildEnv { name = "cuda-merged"; <del> paths = [ (lib.getBin (cudaPackages.cuda_nvcc.__spliced.buildHost or cudaPackages.cuda_nvcc)) (lib.getLib cudaPackages.cuda_cudart) </del> <ins> paths = map lib.getLib cudaLibs ++ [ </ins> (lib.getOutput "static" cudaPackages.cuda_cudart) <del> (lib.getLib cudaPackages.libcublas) </del> <ins> (lib.getBin (cudaPackages.cuda_nvcc.__spliced.buildHost or cudaPackages.cuda_nvcc)) </ins> ]; };

++ lib.optionals enableRocm [ "--suffix LD_LIBRARY_PATH : '${rocmPath}/lib'" "--set-default HIP_PATH '${rocmPath}'" <ins> ] ++ lib.optionals enableCuda [ "--suffix LD_LIBRARY_PATH : '${lib.makeLibraryPath (map lib.getLib cudaLibs)}'" </ins> ]; wrapperArgs = builtins.concatStringsSep " " wrapperOptions;

if enableCuda then buildGoModule.override { stdenv = overrideCC stdenv gcc12; } else buildGoModule; inherit (lib) licenses platforms maintainers; in <del> goBuild ( (lib.optionalAttrs enableRocm { ROCM_PATH = rocmPath; CLBlast_DIR = "${clblast}/lib/cmake/CLBlast"; }) // (lib.optionalAttrs enableCuda { CUDA_LIB_DIR = "${cudaToolkit}/lib"; }) // { inherit pname version src vendorHash ; nativeBuildInputs = [ cmake ] ++ lib.optionals enableRocm [ rocmPackages.llvm.bintools ] ++ lib.optionals enableCuda [ cudaPackages.cuda_nvcc ] ++ lib.optionals (enableRocm || enableCuda) [ makeWrapper autoAddDriverRunpath ] ++ lib.optionals stdenv.isDarwin metalFrameworks; buildInputs = lib.optionals enableRocm (rocmLibs ++ [ libdrm ]) ++ lib.optionals enableCuda [ cudaToolkit cudaPackages.cuda_cudart cudaPackages.cuda_cccl cudaPackages.libcublas ] ++ lib.optionals stdenv.isDarwin metalFrameworks; patches = [ # disable uses of `git` in the `go generate` script # ollama's build script assumes the source is a git repo, but nix removes the git directory # this also disables necessary patches contained in `ollama/llm/patches/` # those patches are applied in `postPatch` ./disable-git.patch ]; postPatch = '' # replace inaccurate version number with actual release version substituteInPlace version/version.go --replace-fail 0.0.0 '${version}' </del> <ins> goBuild { inherit pname version src vendorHash ; env = lib.optionalAttrs enableRocm { ROCM_PATH = rocmPath; CLBlast_DIR = "${clblast}/lib/cmake/CLBlast"; } // lib.optionalAttrs enableCuda { CUDA_LIB_DIR = "${cudaToolkit}/lib"; }; nativeBuildInputs = [ cmake ] ++ lib.optionals enableRocm [ rocmPackages.llvm.bintools ] ++ lib.optionals enableCuda [ cudaPackages.cuda_nvcc ] ++ lib.optionals (enableRocm || enableCuda) [ makeWrapper autoAddDriverRunpath ] ++ lib.optionals stdenv.isDarwin metalFrameworks; buildInputs = lib.optionals enableRocm (rocmLibs ++ [ libdrm ]) ++ lib.optionals enableCuda cudaLibs ++ lib.optionals stdenv.isDarwin metalFrameworks; patches = [ # disable uses of `git` in the `go generate` script # ollama's build script assumes the source is a git repo, but nix removes the git directory # this also disables necessary patches contained in `ollama/llm/patches/` # those patches are applied in `postPatch` ./disable-git.patch ]; </ins> <del> # apply llama.cpp patches for cur in llm/patches/*; do patch -p1 -d llm/llama.cpp < $cur; done </del> <ins> postPatch = '' # replace inaccurate version number with actual release version substituteInPlace version/version.go --replace-fail 0.0.0 '${version}' # apply ollama's patches to `llama.cpp` submodule for diff in llm/patches/*; do patch -p1 -d llm/llama.cpp < $diff done ''; overrideModAttrs = ( finalAttrs: prevAttrs: { # don't run llama.cpp build in the module fetch phase preBuild = ""; } ); preBuild = '' # disable uses of `git`, since nix removes the git directory export OLLAMA_SKIP_PATCHING=true # build llama.cpp libraries for ollama go generate ./... ''; postFixup = '' # the app doesn't appear functional at the moment, so hide it mv "$out/bin/app" "$out/bin/.ollama-app" '' + lib.optionalString (enableRocm || enableCuda) '' # expose runtime libraries necessary to use the gpu wrapProgram "$out/bin/ollama" ${wrapperArgs} </ins> ''; <del> overrideModAttrs = ( finalAttrs: prevAttrs: { # don't run llama.cpp build in the module fetch phase preBuild = ""; } ); preBuild = '' # disable uses of `git`, since nix removes the git directory export OLLAMA_SKIP_PATCHING=true # build llama.cpp libraries for ollama go generate ./... ''; postFixup = '' # the app doesn't appear functional at the moment, so hide it mv "$out/bin/app" "$out/bin/.ollama-app" '' + lib.optionalString (enableRocm || enableCuda) '' # expose runtime libraries necessary to use the gpu wrapProgram "$out/bin/ollama" ${wrapperArgs} ''; ldflags = [ "-s" "-w" "-X=github.com/ollama/ollama/version.Version=${version}" "-X=github.com/ollama/ollama/server.mode=release" ]; </del> <ins> ldflags = [ "-s" "-w" "-X=github.com/ollama/ollama/version.Version=${version}" "-X=github.com/ollama/ollama/server.mode=release" ]; </ins> <del> passthru.tests = { inherit ollama; version = testers.testVersion { inherit version; package = ollama; }; } // lib.optionalAttrs stdenv.isLinux { inherit ollama-rocm ollama-cuda; service = nixosTests.ollama; service-cuda = nixosTests.ollama-cuda; service-rocm = nixosTests.ollama-rocm; </del> <ins> passthru.tests = { inherit ollama; version = testers.testVersion { inherit version; package = ollama; </ins> }; <del> meta = { description = "Get up and running with large language models locally" + lib.optionalString rocmRequested ", using ROCm for AMD GPU acceleration" + lib.optionalString cudaRequested ", using CUDA for NVIDIA GPU acceleration"; homepage = "https://github.com/ollama/ollama"; changelog = "https://github.com/ollama/ollama/releases/tag/v${version}"; license = licenses.mit; platforms = if (rocmRequested || cudaRequested) then platforms.linux else platforms.unix; mainProgram = "ollama"; maintainers = with maintainers; [ abysssol dit7ya elohmeier roydubnium ]; </del> <ins> } // lib.optionalAttrs stdenv.isLinux { inherit ollama-rocm ollama-cuda; service = nixosTests.ollama; service-cuda = nixosTests.ollama-cuda; service-rocm = nixosTests.ollama-rocm; </ins> }; <del> } ) </del> <ins> meta = { description = "Get up and running with large language models locally" + lib.optionalString rocmRequested ", using ROCm for AMD GPU acceleration" + lib.optionalString cudaRequested ", using CUDA for NVIDIA GPU acceleration"; homepage = "https://github.com/ollama/ollama"; changelog = "https://github.com/ollama/ollama/releases/tag/v${version}"; license = licenses.mit; platforms = if (rocmRequested || cudaRequested) then platforms.linux else platforms.unix; mainProgram = "ollama"; maintainers = with maintainers; [ abysssol dit7ya elohmeier roydubnium ]; }; } </ins>