bartowski

/

Liberated-Qwen1.5-7B-GGUF-old

like 2

This only helps, so you can quantize. I don't know why it isn't working. I can't seem to get it to work either. 🤔
Update to the latest version of HuggingFace's Transformers:

pip install -q -U git+https://github.com/huggingface/transformers.git

hmm that's odd, i'll have to try it with llama.cpp to see if the quant itself is busted or if something is off in ollama

can't run with llmstudio which is based on llama.cpp.same error code :

{
  "cause": "llama.cpp error: 'invalid unordered_map<K, T> key'",
  "errorData": {
    "n_ctx": 2048,
    "n_batch": 512,
    "n_gpu_layers": 0
  },
  "data": {
    "memory": {
      "ram_capacity": "15.68 GB",
      "ram_unused": "319.91 MB"
    },
    "gpu": {
      "type": "NvidiaCuda",
      "vram_recommended_capacity": "4.00 GB",
      "vram_unused": "3.25 GB"
    },
    "os": {
      "platform": "win32",
      "version": "10.0.19045",
      "supports_avx2": true
    },
    "app": {
      "version": "0.2.14",
      "downloadsDir": "D:\\llmmodels"
    },
    "model": {}
  },
  "title": "Failed to load model",
  "systemDiagnostics": {
    "memory": {
      "ram_capacity": 16839671808,
      "ram_unused": 510345216
    },
    "gpu": {
      "type": "NvidiaCuda",
      "vram_recommended_capacity": 4294639616,
      "vram_unused": 3490290176
    },
    "os": {
      "platform": "win32",
      "version": "10.0.19045",
      "supports_avx2": true
    },
    "app": {
      "version": "0.2.14",
      "downloadsDir": "D:\\llmmodels"
    },
    "model": {
      "gguf_preview": {
        "name": "models",
        "arch": "llama",
        "quant": "Q5_K_M",
        "context_length": 32768,
        "embedding_length": 4096,
        "num_layers": 32,
        "rope": {
          "freq_base": 1000000,
          "dimension_count": 128
        },
        "head_count": 32,
        "head_count_kv": 32,
        "parameters": "7B"
      },
      "filesize": 5528541024,
      "config": {
        "path": "D:\\llmmodels\\alibaba\\qwen1_57b-liberated\\Liberated-Qwen1.5-7B-Q5_K_M.gguf",
        "loadConfig": {
          "n_ctx": 2048,
          "n_batch": 512,
          "rope_freq_base": 0,
          "rope_freq_scale": 0,
          "n_gpu_layers": 0,
          "use_mlock": true,
          "main_gpu": 0,
          "tensor_split": [
            0
          ],
          "seed": -1,
          "f16_kv": true,
          "use_mmap": true,
          "no_kv_offload": false,
          "num_experts_used": 0
        }
      }
    }
  }
}```

Yeah I'm getting issues too, talking to with Eric about it