Repository: ggml-org/llama.cpp
Pull request: Implement GGML_CPU_ALL_VARIANTS for PowerPC

This first draft follows the recent ARM approach and should technically be OK though far from perfect. It introduces the platform into the backend scoring, but in a "dumb" way, scoring it like any other on/off feature. I have an improvement for this in the works but this improvement needs to be implemented for all architectures building variants (x86, ARM) at the same time, so that will be a follow-up PR.

However this PowerPC build runs into `SIGILL` as soon as a backend built for a newer architecture than supported by the current CPU is loaded. From the looks of it,  I think this is the scenario that @slaren [commented](https://github.com/ggml-org/llama.cpp/pull/14049#issuecomment-2952606532) in #14049: the compiler sees certain instructions enabled and attempts to use them during initialization at [repack.cpp#1401](llama.cpp/blob/master/ggml/src/ggml-cpu/repack.cpp#L1401), even if the code itself doesn't use intrinsics yet at that point. So the overall program crashes before we can even "kick" the backend out as unsupported.

If my interpretation is right, then this is a general `DL_BACKEND` issue that only just happens to manifest itself with PowerPC so far.

I'm not yet familiar with that part of the code so I don't see an obvious solution. If anyone has an idea, I would appreciate it. My first instinct would be to separate this part from the code from the scoring, but I would expect that to add to complexity, perhaps  there is a simpler solution to the initialization above that I'm just missing.

Backtrace:

```c++
Using host libthread_db library "/lib/powerpc64le-linux-gnu/libthread_db.so.1".
ggml_backend_load_best: /home/ckk/llama.cpp-le/build/bin/libggml-cpu-power7_2.so score: 67
ggml_backend_load_best: /home/ckk/llama.cpp-le/build/bin/libggml-cpu-power8_1.so score: 5
ggml_backend_load_best: /home/ckk/llama.cpp-le/build/bin/libggml-cpu-power7_1.so score: 3
ggml_backend_load_best: /home/ckk/llama.cpp-le/build/bin/libggml-cpu-power9.so score: 73
ggml_backend_load_best: /home/ckk/llama.cpp-le/build/bin/libggml-cpu-power8_2.so score: 69

Program received signal SIGILL, Illegal instruction.
__static_initialization_and_destruction_0 () at /home/ckk/llama.cpp-le/ggml/src/ggml-cpu/repack.cpp:1401
1401    static const tensor_traits<block_q4_0, 4, 4, GGML_TYPE_Q8_0> q4_0_4x4_q8_0;
(gdb) bt
#0  __static_initialization_and_destruction_0 () at /home/ckk/llama.cpp-le/ggml/src/ggml-cpu/repack.cpp:1401
#1  0x00007ffff520ec00 in _GLOBAL__sub_I_repack.cpp(void) () at /home/ckk/llama.cpp-le/ggml/src/ggml-cpu/repack.cpp:1555
#2  0x00007ffff7f96b0c in call_init (l=<optimized out>, argc=<optimized out>, argv=<optimized out>, env=<optimized out>)
    at dl-init.c:74
#3  _dl_init (main_map=0x1004f7250, argc=2, argv=0x7ffffffff378, env=0x7ffffffff390) at dl-init.c:121
#4  0x00007ffff7faa28c in call_dl_init (closure=<optimized out>, closure@entry=0x7ffffffde210) at dl-open.c:493
#5  0x00007ffff7f916e0 in __GI__dl_catch_exception (exception=exception@entry=0x0, operate=<optimized out>, 
    operate@entry=0x7ffff7faa260 <call_dl_init>, args=<optimized out>, args@entry=0x7ffffffde210) at dl-catch.c:215
#6  0x00007ffff7faa41c in dl_open_worker (a=0x7ffffffde210) at dl-open.c:799
#7  dl_open_worker (a=a@entry=0x7ffffffde210) at dl-open.c:750
#8  0x00007ffff7f9163c in __GI__dl_catch_exception (exception=exception@entry=0x7ffffffde258, 
    operate=operate@entry=0x7ffff7faa2f0 <dl_open_worker>, args=args@entry=0x7ffffffde210) at dl-catch.c:241
#9  0x00007ffff7fabccc in _dl_open (file=0x1004fb0d0 "/home/ckk/llama.cpp-le/build/bin/libggml-cpu-power10.so", 
    mode=-2147483646, caller_dlopen=0x7ffff7f196bc <dl_load_library(std::filesystem::__cxx11::path const&)+84>, nsid=-2, 
    argc=2, argv=0x7ffffffff378, env=0x7ffffffff390) at dl-open.c:874
#10 0x00007ffff6aa73f4 in dlopen_doit (a=a@entry=0x7ffffffde6e8) at dlopen.c:56
#11 0x00007ffff7f9163c in __GI__dl_catch_exception (exception=exception@entry=0x7ffffffde600, 
    operate=0x7ffff6aa7370 <dlopen_doit>, args=0x7ffffffde6e8) at dl-catch.c:241
#12 0x00007ffff7f917bc in _dl_catch_error (objname=objname@entry=0x7ffffffde678, errstring=errstring@entry=0x7ffffffde680, 
    mallocedp=mallocedp@entry=0x7ffffffde677, operate=<optimized out>, args=<optimized out>) at dl-catch.c:260
#13 0x00007ffff6aa6cd8 in _dlerror_run (operate=<optimized out>, operate@entry=0x7ffff6aa7370 <dlopen_doit>, 
    args=<optimized out>, args@entry=0x7ffffffde6e8) at dlerror.c:138
#14 0x00007ffff6aa7504 in dlopen_implementation (file=<optimized out>, mode=<optimized out>, dl_caller=<optimized out>)
    at dlopen.c:71
#15 ___dlopen (file=<optimized out>, mode=<optimized out>) at dlopen.c:81
#16 0x00007ffff7f196bc in dl_load_library (
    path=filesystem::path "/home/ckk/llama.cpp-le/build/bin/libggml-cpu-power10.so" = {...})
    at /home/ckk/llama.cpp-le/ggml/src/ggml-backend-reg.cpp:140
#17 0x00007ffff7f1ab14 in ggml_backend_load_best (name=0x7ffff7f2c780 "cpu", silent=false, user_search_path=0x0)
    at /home/ckk/llama.cpp-le/ggml/src/ggml-backend-reg.cpp:517
#18 0x00007ffff7f1b570 in ggml_backend_load_all_from_path (dir_path=0x0)
    at /home/ckk/llama.cpp-le/ggml/src/ggml-backend-reg.cpp:580
#19 0x00007ffff7f1b400 in ggml_backend_load_all () at /home/ckk/llama.cpp-le/ggml/src/ggml-backend-reg.cpp:559
#20 0x00000001000c0308 in common_params_parser_init (params=..., ex=LLAMA_EXAMPLE_MAIN, 
    print_usage=0x1000850f8 <print_usage(int, char**)>) at /home/ckk/llama.cpp-le/common/arg.cpp:1225
#21 0x00000001000ae54c in common_params_parse (argc=2, argv=0x7ffffffff378, params=..., ex=LLAMA_EXAMPLE_MAIN, 
    print_usage=0x1000850f8 <print_usage(int, char**)>) at /home/ckk/llama.cpp-le/common/arg.cpp:1180
#22 0x0000000100085900 in main (argc=2, argv=0x7ffffffff378) at /home/ckk/llama.cpp-le/tools/main/main.cpp:89
```

I was testing this on POWER8 big-endian, and POWER9 little-endian (the porter boxes that Debian has available). Test command was `llama-cli --version`.

Changed files:
filename: ggml/src/CMakeLists.txt
patch: @@ -286,6 +286,10 @@ function(ggml_add_cpu_backend_variant tag_name)
         foreach (feat ${ARGN})
             set(GGML_INTERNAL_${feat} ON)
         endforeach()
+    elseif (GGML_SYSTEM_ARCH STREQUAL "PowerPC")
+        foreach (feat ${ARGN})
+            set(GGML_INTERNAL_${feat} ON)
+        endforeach()
     endif()
 
     ggml_add_cpu_backend_variant_impl(${tag_name})
@@ -337,6 +341,19 @@ if (GGML_CPU_ALL_VARIANTS)
         else()
             message(FATAL_ERROR "Unsupported ARM target OS: ${CMAKE_SYSTEM_NAME}")
         endif()
+    elseif (GGML_SYSTEM_ARCH STREQUAL "PowerPC")
+        if (CMAKE_SYSTEM_NAME MATCHES "Linux")
+            ggml_add_cpu_backend_variant(power0)
+            ggml_add_cpu_backend_variant(power7_1       POWER7)
+            ggml_add_cpu_backend_variant(power7_2       POWER7  VSX)
+            ggml_add_cpu_backend_variant(power8_1       POWER8)
+            ggml_add_cpu_backend_variant(power8_2       POWER8  VSX)
+            ggml_add_cpu_backend_variant(power9         POWER9  VSX)
+            ggml_add_cpu_backend_variant(power10        POWER10 VSX)
+            ggml_add_cpu_backend_variant(power11        POWER11 VSX)
+        else()
+            message(FATAL_ERROR "Unsupported PowerPC target OS: ${CMAKE_SYSTEM_NAME}")
+        endif()
     else()
         message(FATAL_ERROR "GGML_CPU_ALL_VARIANTS not yet supported with ${GGML_SYSTEM_ARCH} on ${CMAKE_SYSTEM_NAME}")
     endif()

filename: ggml/src/ggml-cpu/CMakeLists.txt
patch: @@ -388,6 +388,27 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
             else()
                 list(APPEND ARCH_FLAGS -mcpu=native -mtune=native -mpowerpc64)
             endif()
+        elseif(GGML_CPU_ALL_VARIANTS)
+            # Begin with the lowest baseline
+            set(ARCH_DEFINITIONS "")
+
+            # When a feature is selected, bump the MCPU to the first
+            # version that supported it
+            foreach(PVER RANGE 7 11)
+                if(DEFINED GGML_INTERNAL_POWER${PVER})
+                    set(POWERPC_MCPU "power${PVER}")
+                    list(APPEND ARCH_DEFINITIONS GGML_USE_POWER${PVER})
+                endif()
+            endforeach()
+            if (GGML_INTERNAL_VSX)
+                list(APPEND ARCH_DEFINITIONS GGML_USE_VSX)
+                list(APPEND ARCH_FLAGS -mvsx)
+            endif()
+
+            if (DEFINED POWERPC_MCPU)
+                list(APPEND ARCH_FLAGS -mcpu=${POWERPC_MCPU})
+            endif()
+            ggml_add_cpu_backend_features(${GGML_CPU_NAME} powerpc ${ARCH_DEFINITIONS})
         else()
             if (GGML_CPU_POWERPC_CPUTYPE)
                 list(APPEND ARCH_FLAGS -mcpu=${GGML_CPU_POWERPC_CPUTYPE})

filename: ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp
patch: @@ -0,0 +1,82 @@
+# include "ggml-backend-impl.h"
+
+#if defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__)
+
+#if defined(__linux__)
+#include <sys/auxv.h>
+#endif
+
+#include <string>
+
+struct powerpc_features {
+    std::string platform = "";
+    int power_version    = -1;
+
+    bool has_vsx         = false;
+
+    powerpc_features() {
+#if defined(__linux__)
+        unsigned long auxval = getauxval(AT_PLATFORM);
+        if (auxval) {
+            platform = std::string(reinterpret_cast<const char*>(auxval));
+            // TBD: Do systems exist that return this in uppercase?
+            if (platform.substr(0, 5) == "power") {
+                // Extractt a numeric suffix, if one exists
+                int vpos = -1;
+                for (int i = platform.length() - 1; i >= 0; i--) {
+                    if (std::isdigit(platform[i])) {
+                        vpos = i;
+                    } else {
+                        break;
+                    }
+                }
+                if (vpos > -1) {
+                    power_version = std::stoi(platform.substr(vpos));
+                }
+            }
+        }
+#endif
+        if (power_version >= 9) {
+            has_vsx = true;
+        }
+    }
+};
+
+static int ggml_backend_cpu_powerpc_score() {
+    int score = 1;
+    powerpc_features pf;
+
+// Platform scores
+#if defined(GGML_USE_POWER7)
+    if (pf.power_version < 7) { return 0; }
+    score += 1<<1;
+#endif
+#if defined(GGML_USE_POWER8)
+    if (pf.power_version < 8) { return 0; }
+    score += 1<<2;
+#endif
+#if defined(GGML_USE_POWER9)
+    if (pf.power_version < 9) { return 0; }
+    score += 1<<3;
+#endif
+#if defined(GGML_USE_POWER10)
+    if (pf.power_version < 10) { return 0; }
+    score += 1<<4;
+#endif
+#if defined(GGML_USE_POWER11)
+    if (pf.power_version < 11) { return 0; }
+    score += 1<<5;
+#endif
+
+// Feature scores
+#if defined(GGML_USE_VSX)
+    if (!pf.has_vsx) { return 0; }
+    score += 1<<6;
+#endif
+
+    return score;
+}
+
+GGML_BACKEND_DL_SCORE_IMPL(ggml_backend_cpu_powerpc_score)
+
+#endif // defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__)

filename: ggml/src/ggml-cpu/repack.cpp
patch: @@ -1411,44 +1411,45 @@ template <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS, ggml_type PAR
     }
 };
 
-// instance for Q4
-static const tensor_traits<block_q4_0, 4, 4, GGML_TYPE_Q8_0> q4_0_4x4_q8_0;
-static const tensor_traits<block_q4_0, 8, 4, GGML_TYPE_Q8_0> q4_0_4x8_q8_0;
-static const tensor_traits<block_q4_0, 8, 8, GGML_TYPE_Q8_0> q4_0_8x8_q8_0;
-static const tensor_traits<block_q4_K, 8, 8, GGML_TYPE_Q8_K> q4_K_8x8_q8_K;
-
-// instance for IQ4
-static const tensor_traits<block_iq4_nl, 4, 4, GGML_TYPE_Q8_0> iq4_nl_4x4_q8_0;
-
 }  // namespace ggml::cpu::repack
 
 static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(const struct ggml_tensor * cur) {
+
+    // instance for Q4
+    static const ggml::cpu::repack::tensor_traits<block_q4_0, 4, 4, GGML_TYPE_Q8_0> q4_0_4x4_q8_0;
+    static const ggml::cpu::repack::tensor_traits<block_q4_0, 8, 4, GGML_TYPE_Q8_0> q4_0_4x8_q8_0;
+    static const ggml::cpu::repack::tensor_traits<block_q4_0, 8, 8, GGML_TYPE_Q8_0> q4_0_8x8_q8_0;
+    static const ggml::cpu::repack::tensor_traits<block_q4_K, 8, 8, GGML_TYPE_Q8_K> q4_K_8x8_q8_K;
+
+    // instance for IQ4
+    static const ggml::cpu::repack::tensor_traits<block_iq4_nl, 4, 4, GGML_TYPE_Q8_0> iq4_nl_4x4_q8_0;
+
     if (cur->type == GGML_TYPE_Q4_0) {
         if (ggml_cpu_has_avx2() || (ggml_cpu_has_sve() && ggml_cpu_has_matmul_int8() && ggml_cpu_get_sve_cnt() == QK8_0)) {
             if (cur->ne[1] % 8 == 0) {
-                return &ggml::cpu::repack::q4_0_8x8_q8_0;
+                return &q4_0_8x8_q8_0;
             }
         }
         if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) {
             if (cur->ne[1] % 4 == 0) {
-                return &ggml::cpu::repack::q4_0_4x8_q8_0;
+                return &q4_0_4x8_q8_0;
             }
         }
         if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) {
             if (cur->ne[1] % 4 == 0) {
-                return &ggml::cpu::repack::q4_0_4x4_q8_0;
+                return &q4_0_4x4_q8_0;
             }
         }
     } else if (cur->type == GGML_TYPE_Q4_K) {
         if (ggml_cpu_has_avx2()) {
             if (cur->ne[1] % 8 == 0) {
-                return &ggml::cpu::repack::q4_K_8x8_q8_K;
+                return &q4_K_8x8_q8_K;
             }
         }
     } else if (cur->type == GGML_TYPE_IQ4_NL) {
         if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) {
             if (cur->ne[1] % 4 == 0) {
-                return &ggml::cpu::repack::iq4_nl_4x4_q8_0;
+                return &iq4_nl_4x4_q8_0;
             }
         }
     }