Repository: ggml-org/llama.cpp
Pull request: ggml-hexagon: graceful fallback for older socs where rpcmem_alloc2 and FASTRPC_GET_URI is unsupported

Older SoCs like the Gen2 does not seem to support obtaining session URIs via `FASTRPC_GET_URI`. Additionally, they do not support the new `rpcmem_alloc2`, and should use `rpcmem_alloc` instead.

This PR adds graceful fallback for older SoCs:
1. dynamically load `rpcmem_alloc2` instead of statically loading, fall back to `rpcmem_alloc` if unable to find `alloc2`
2. added fallback for older SoCs which do not support `FASTRPC_GET_URI`, falls back to single session uri

Previously, `ggml-hexagon` quits when running on an SD8 Gen2, which does not support `alloc2` nor allow creating multiple sessions.

This PR is tested to work on both consumer S23 Ultra (SD8 Gen2), and S25+ Ultra (SD8 Gen4).

@max-krasnyansky I would appreciate your direction on if this PR is the way to go forward providing graceful fallback for older SoCs?
1. ~~`libcdsprpc.so` library name is hard-coded, is this a problem?~~
2. undefined results if you enable multiple devices (i.e. HTP0, HTP1) on Gen2, all the opened session URIs are the same, but `ggml-hexagon` still thinks multiple session are open, should this be the end-users responsibility to not open multiple sessions on devices that do not support it?
3. ~~function pointer type to `rpcmem_allow[2]` is hardcoded, and declared as static, `dlopen` and `dlclose` happens in `ggml_hexagon_registry`, from my understanding, it seems it's guaranteed to only initialise once~~
4. I do not have a Windows Snapdragon device to test, but theoretically porting this to windows should be a simple as conditionally compiling the `dlopen` and `dlclose` part to use `.dll` instead

EDIT: thanks to @chraac in here: https://github.com/ggml-org/llama.cpp/issues/16911#issuecomment-3489159753, updated PR to use weak symbols instead of dynamically loading `libcdsprpc.so`, which is a much more elegant solution

Changed files:
filename: ggml/src/ggml-hexagon/ggml-hexagon.cpp
patch: @@ -367,7 +367,13 @@ struct ggml_backend_hexagon_buffer_context {
     ggml_backend_hexagon_buffer_context(ggml_hexagon_session * sess, size_t size, bool repack) {
         size += 4 * 1024;  // extra page for padding
 
-        this->base = (uint8_t *) rpcmem_alloc2(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS | RPCMEM_HEAP_NOREG, size);
+        if (rpcmem_alloc2) {
+            this->base = (uint8_t *) rpcmem_alloc2(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS | RPCMEM_HEAP_NOREG, size);
+        } else {
+            GGML_LOG_INFO("ggml-hex: %s rpcmem_alloc2 not found, falling back to rpcmem_alloc\n", sess->name.c_str());
+            this->base = (uint8_t *) rpcmem_alloc(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS | RPCMEM_HEAP_NOREG, size);
+        }
+
         if (!this->base) {
             GGML_LOG_ERROR("ggml-hex: %s failed to allocate buffer : size %zu\n", sess->name.c_str(), size);
             throw std::runtime_error("ggml-hex: rpcmem_alloc failed (see log for details)");
@@ -1679,12 +1685,13 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) {
     }
 
     // Get session URI
-    char htp_uri[256];
-    sprintf(htp_uri, "file:///libggml-htp-v%u.so?htp_iface_skel_handle_invoke&_modver=1.0", opt_arch);
 
     char session_uri[256];
     {
-        struct remote_rpc_get_uri u;
+        char htp_uri[256];
+        snprintf(htp_uri, sizeof(htp_uri), "file:///libggml-htp-v%u.so?htp_iface_skel_handle_invoke&_modver=1.0", opt_arch);
+
+        struct remote_rpc_get_uri u = {};
         u.session_id      = this->session_id;
         u.domain_name     = const_cast<char *>(CDSP_DOMAIN_NAME);
         u.domain_name_len = strlen(CDSP_DOMAIN_NAME);
@@ -1695,8 +1702,12 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) {
 
         int err = remote_session_control(FASTRPC_GET_URI, (void *) &u, sizeof(u));
         if (err != AEE_SUCCESS) {
-            GGML_LOG_ERROR("ggml-hex: failed to get URI for session %d : error 0x%x\n", dev_id, err);
-            throw std::runtime_error("ggml-hex: remote_session_control(get-uri) failed (see log for details)");
+            // fallback to single session uris
+            int htp_URI_domain_len = strlen(htp_uri) + MAX_DOMAIN_NAMELEN;
+
+            snprintf(session_uri, htp_URI_domain_len, "%s%s", htp_uri, my_domain->uri);
+
+            GGML_LOG_WARN("ggml-hex: failed to get URI for session %d : error 0x%x. Falling back to single session URI: %s\n", dev_id, err, session_uri);
         }
     }
 
@@ -3668,6 +3679,11 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) {
         }
     }
 
+    if(opt_arch < 75) {
+        opt_ndev = 1;
+        GGML_LOG_WARN("ggml-hex: forcing ndev to 1 for SoCs archs lower than v75.\n");
+    }
+
     GGML_LOG_INFO("ggml-hex: Hexagon Arch version v%d\n", opt_arch);
 
     // Create devices / sessions

filename: ggml/src/ggml-hexagon/htp-utils.h
patch: @@ -64,6 +64,7 @@ extern "C" {
 #    pragma weak remote_handle64_control
 #    pragma weak fastrpc_mmap
 #    pragma weak fastrpc_munmap
+#    pragma weak rpcmem_alloc2
 #endif
 
 #if !defined(_WINDOWS)