cjpais

/

llava-v1.6-34B-gguf

like 34

Loading with llama_cpp_python, you get this error:

gguf_init_from_file: invalid magic characters ''
llama_model_load: error loading model: llama_model_loader: failed to load model from /share/whirl/repos/llava-v1.6-34B-gguf

llama_load_model_from_file: failed to load model

{
    "name": "AssertionError",
    "message": "",
    "stack": "---------------------------------------------------------------------------
AssertionError                            Traceback (most recent call last)
Cell In[1], line 6
      4 MODEL_PATH=\"/repos/llava-v1.6-34B-gguf\"
----> 6 llm = Llama(model_path=MODEL_PATH, chat_format=\"chatml\")
      7 llm.create_chat_completion(
      8     messages=[
      9         {
   (...)
     23     temperature=0.7,
     24 )

File /.venv/lib/python3.10/site-packages/llama_cpp/llama.py:296, in Llama.__init__(self, model_path, n_gpu_layers, split_mode, main_gpu, tensor_split, vocab_only, use_mmap, use_mlock, kv_overrides, seed, n_ctx, n_batch, n_threads, n_threads_batch, rope_scaling_type, rope_freq_base, rope_freq_scale, yarn_ext_factor, yarn_attn_factor, yarn_beta_fast, yarn_beta_slow, yarn_orig_ctx, mul_mat_q, logits_all, embedding, offload_kqv, last_n_tokens_size, lora_base, lora_scale, lora_path, numa, chat_format, chat_handler, draft_model, verbose, **kwargs)
    293     self.context_params.n_ctx = self._model.n_ctx_train()
    294     self.context_params.n_batch = self.n_batch
--> 296 self._ctx = _LlamaContext(
    297     model=self._model,
    298     params=self.context_params,
    299     verbose=self.verbose,
    300 )
    302 self._batch = _LlamaBatch(
    303     n_tokens=self.n_batch,
    304     embd=0,
    305     n_seq_max=self.context_params.n_ctx,
    306     verbose=self.verbose,
    307 )
    309 if self.lora_path:

File /.venv/lib/python3.10/site-packages/llama_cpp/_internals.py:252, in _LlamaContext.__init__(self, model, params, verbose)
    248 self.verbose = verbose
    250 self._llama_free = llama_cpp._lib.llama_free  # type: ignore
--> 252 assert self.model.model is not None
    254 self.ctx = llama_cpp.llama_new_context_with_model(
    255     self.model.model, self.params
    256 )

AssertionError: "
}