Repository: carstenbauer/ThreadPinning.jl (julia)

Issue:
Could not load library libmkl_rt
I'm trying to use this package on a cluster, and came across the following error:
```julia
julia> using MKL, ThreadPinning

julia> threadinfo(; blas=true, hints=true)

| 0,1,2,3,4,5,6,14,15,16,17,18,19,20 | 7,8,9,10,11,12,13,21,22,23,24,25,26,27 | 

# = Julia thread, | = Socket seperator

Julia threads: 14
├ Occupied CPU-threads: 14
└ Mapping (Thread => CPUID): 1 => 6, 2 => 14, 3 => 15, 4 => 16, 5 => 18, ...

BLAS: libmkl_rt.so
├ mkl_get_num_threads: 2
ERROR: could not load library "libmkl_rt"
libmkl_rt.so: cannot open shared object file: No such file or directory
Stacktrace:
 [1] dlopen(s::String, flags::UInt32; throw_error::Bool)
   @ Base.Libc.Libdl ./libdl.jl:117
 [2] dlopen (repeats 2 times)
   @ ./libdl.jl:116 [inlined]
 [3] dlpath(libname::String)
   @ Base.Libc.Libdl ./libdl.jl:240
 [4] mkl_get_dynamic
   @ /scratch/user/.julia/packages/ThreadPinning/UMdJD/src/mkl.jl:26 [inlined]
 [5] threadinfo(; blas::Bool, hints::Bool, color::Bool, masks::Bool, groupby::Symbol, threadpool::Symbol, kwargs::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
   @ ThreadPinning /scratch/user/.julia/packages/ThreadPinning/UMdJD/src/threadinfo.jl:95
 [6] top-level scope
   @ REPL[2]:1

julia> using LinearAlgebra

julia> BLAS.get_config()
LinearAlgebra.BLAS.LBTConfig
Libraries: 
├ [ILP64] libmkl_rt.so
└ [ LP64] libmkl_rt.so

julia> versioninfo()
Julia Version 1.8.4
Commit 00177ebc4fc (2022-12-23 21:32 UTC)
Platform Info:
  OS: Linux (x86_64-linux-gnu)
  CPU: 28 × Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GHz
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-13.0.1 (ORCJIT, broadwell)
  Threads: 14 on 28 virtual cores
Environment:
  LD_LIBRARY_PATH = /home/user/lib:/lib:/home/user/lib:/lib::/home/user/.local/lib
  JULIA_DEPOT_PATH = /scratch/user/.julia
  JULIA_REVISE_POLL = 1
  JULIA_NUM_PRECOMPILE_TASKS = 1

(RossbyWaveSpectrum) pkg> st MKL
Project RossbyWaveSpectrum v0.1.0
Status `/scratch/user/RossbyWaveSpectrum.jl/Project.toml`
  [33e6dc65] MKL v0.6.0
```

I remember this package working on a previous Julia version on this cluster long back.

Pull request:
Closes #48

The issue was that `dlpath` errors despite having done `using MKL` (presumably because the artefact, `libmkl_rt`, doesn't go onto the `LD_LIBRARY_PATH`). Fixed it by querying the full path to `libmkl_rt` from `BLAS.get_config()` on Julia >= 1.7.

Patch:
diff --git a/Project.toml b/Project.toml
index fde7b2f7..a07d2e2b 100644
--- a/Project.toml
+++ b/Project.toml
@@ -12,16 +12,18 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 SnoopPrecompile = "66db9d55-30c0-4569-8b51-7e840670fc0c"
 
 [compat]
+MKL = "0.6"
 Preferences = "1"
 SnoopPrecompile = "1"
 TestItemRunner = "0.2"
 julia = "1.6"
 
 [extras]
+MKL = "33e6dc65-8f57-5167-99aa-e5a354878fb2"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
 
 [targets]
-test = ["Test", "Random", "TestItemRunner", "Statistics"]
+test = ["Test", "Random", "TestItemRunner", "Statistics", "MKL"]
diff --git a/src/mkl.jl b/src/mkl.jl
index ade82602..683d37cc 100644
--- a/src/mkl.jl
+++ b/src/mkl.jl
@@ -1,34 +1,61 @@
 """
-Check if MKL library (`libmkl_rt`) is available in `Libdl.dllist()` (as is the case when
-loading [MKL.jl](https://github.com/JuliaLinearAlgebra/MKL.jl) or
-[MKL_jll.jl](https://github.com/JuliaBinaryWrappers/MKL_jll.jl)).
+Check whether Intel MKL is currently loaded via libblastrampoline (Julia >= 1.7)
+or is available in `Libdl.dllist()` (Julia 1.6).
 """
-function mkl_is_available()
-    any(endswith(lib, "libmkl_rt.$(Libdl.dlext)") for lib in Libdl.dllist())
+function mkl_is_loaded()
+    @static if VERSION <= v"1.7-"
+        any(endswith(lib, "libmkl_rt.$(Libdl.dlext)") for lib in Libdl.dllist())
+    else
+        any(x -> startswith(x, "libmkl_rt"),
+            basename(lib.libname) for lib in BLAS.get_config().loaded_libs)
+    end
 end
 
 """
-Check whether Intel MKL is currently loaded (via libblastrampoline).
+Returns the full path to the `libmkl_rt` library if the latter is loaded. Will try to
+locate the library and, if successfull, will cache the result. Throws an error otherwise.
+
+To force an update of the cache, provide `force_update=true`.
 """
-function mkl_is_loaded()
+function mkl_fullpath(; force_update = false)
+    if isnothing(MKL_PATH[]) || force_update
+        mklpath = _find_mkl()
+        MKL_PATH[] = mklpath
+    end
+    return MKL_PATH[]
+end
+
+# locate libmkl_rt (i.e. full path to it)
+function _find_mkl()
     @static if VERSION <= v"1.7-"
-        mkl_is_available()
+        mklidx = findfirst(lib -> endswith(lib, "libmkl_rt.$(Libdl.dlext)"), Libdl.dllist())
+        if isnothing(mklidx)
+            error("Intel MKL not loaded.")
+        end
+        fullpath = Libdl.dllist()[mklidx]
     else
-        any(x -> startswith(x, "libmkl_rt"),
-            basename(lib.libname) for lib in BLAS.get_config().loaded_libs)
+        mklidx = findfirst(lib -> startswith(basename(lib.libname), "libmkl_rt"),
+                           BLAS.get_config().loaded_libs)
+        if isnothing(mklidx)
+            error("Intel MKL not loaded via LBT.")
+        end
+        fullpath = BLAS.get_config().loaded_libs[mklidx].libname
     end
+    return fullpath
 end
 
 """
     mkl_get_dynamic()
 Wrapper around the MKL function [`mkl_get_dynamic`](https://www.intel.com/content/www/us/en/develop/documentation/onemkl-developer-reference-fortran/top/support-functions/threading-control/mkl-get-dynamic.html).
 """
-mkl_get_dynamic() = @ccall dlpath("libmkl_rt").mkl_get_dynamic()::Cint
+mkl_get_dynamic() = @ccall mkl_fullpath().mkl_get_dynamic()::Cint
 
 """
     mkl_set_dynamic(flag::Integer)
 Wrapper around the MKL function [`mkl_set_dynamic`](https://www.intel.com/content/www/us/en/develop/documentation/onemkl-developer-reference-c/top/support-functions/threading-control/mkl-set-dynamic.html).
 """
 function mkl_set_dynamic(flag::Integer)
-    @ccall dlpath("libmkl_rt").MKL_Set_Dynamic(flag::Cint)::Cvoid
+    @ccall mkl_fullpath().MKL_Set_Dynamic(flag::Cint)::Cvoid
 end
+
+const MKL_PATH = Ref{Union{Nothing, String}}(nothing)


Test patch:
diff --git a/test/mkl_test.jl b/test/mkl_test.jl
new file mode 100644
index 00000000..d84cdd37
--- /dev/null
+++ b/test/mkl_test.jl
@@ -0,0 +1,17 @@
+using ThreadPinning
+using Test
+using MKL
+
+@testset "MKL utilities" begin
+    @test ThreadPinning.mkl_is_loaded()
+    @test ThreadPinning.mkl_get_dynamic() isa Integer
+    @test isnothing(ThreadPinning.mkl_set_dynamic(1))
+    @test ThreadPinning.mkl_get_dynamic() == 1
+    @test isnothing(ThreadPinning.mkl_set_dynamic(0))
+    @test ThreadPinning.mkl_get_dynamic() == 0
+end
+
+@testset "MKL threadinfo" begin
+    @test isnothing(threadinfo())
+    @test isnothing(threadinfo(; blas = true, hints = true))
+end
diff --git a/test/runtests.jl b/test/runtests.jl
index 4b935693..b79b7dd9 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -15,3 +15,4 @@ Threads.nthreads() ≥ 4 ||
 @testitem "likwid-pin" begin include("likwid-pin_test.jl") end
 @testitem "openblas" begin include("openblas_test.jl") end
 @testitem "core2core latency" begin include("latency_test.jl") end
+@testitem "intel mkl" begin include("mkl_test.jl") end