|
main: build = 3010 (95f84d5c) |
|
main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu |
|
main: seed = 1716843987 |
|
llama_model_loader: loaded meta data with 26 key-value pairs and 563 tensors from internlm2-math-plus-mixtral8x22b-IMat-GGUF/internlm2-math-plus-mixtral8x22b.gguf (version GGUF V3 (latest)) |
|
llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. |
|
llama_model_loader: - kv 0: general.architecture str = llama |
|
llama_model_loader: - kv 1: general.name str = internlm2-math-plus-mixtral8x22b |
|
llama_model_loader: - kv 2: llama.block_count u32 = 56 |
|
llama_model_loader: - kv 3: llama.context_length u32 = 65536 |
|
llama_model_loader: - kv 4: llama.embedding_length u32 = 6144 |
|
llama_model_loader: - kv 5: llama.feed_forward_length u32 = 16384 |
|
llama_model_loader: - kv 6: llama.attention.head_count u32 = 48 |
|
llama_model_loader: - kv 7: llama.attention.head_count_kv u32 = 8 |
|
llama_model_loader: - kv 8: llama.rope.freq_base f32 = 1000000.000000 |
|
llama_model_loader: - kv 9: llama.attention.layer_norm_rms_epsilon f32 = 0.000010 |
|
llama_model_loader: - kv 10: llama.expert_count u32 = 8 |
|
llama_model_loader: - kv 11: llama.expert_used_count u32 = 2 |
|
llama_model_loader: - kv 12: general.file_type u32 = 1 |
|
llama_model_loader: - kv 13: llama.vocab_size u32 = 32064 |
|
llama_model_loader: - kv 14: llama.rope.dimension_count u32 = 128 |
|
llama_model_loader: - kv 15: tokenizer.ggml.model str = llama |
|
llama_model_loader: - kv 16: tokenizer.ggml.pre str = default |
|
llama_model_loader: - kv 17: tokenizer.ggml.tokens arr[str,32064] = ["<unk>", "<s>", "</s>", "<0x00>", "<... |
|
llama_model_loader: - kv 18: tokenizer.ggml.scores arr[f32,32064] = [0.000000, 0.000000, 0.000000, 0.0000... |
|
llama_model_loader: - kv 19: tokenizer.ggml.token_type arr[i32,32064] = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ... |
|
llama_model_loader: - kv 20: tokenizer.ggml.bos_token_id u32 = 1 |
|
llama_model_loader: - kv 21: tokenizer.ggml.eos_token_id u32 = 2 |
|
llama_model_loader: - kv 22: tokenizer.ggml.unknown_token_id u32 = 0 |
|
llama_model_loader: - kv 23: tokenizer.ggml.add_bos_token bool = true |
|
llama_model_loader: - kv 24: tokenizer.ggml.add_eos_token bool = false |
|
llama_model_loader: - kv 25: general.quantization_version u32 = 2 |
|
llama_model_loader: - type f32: 169 tensors |
|
llama_model_loader: - type f16: 394 tensors |
|
llm_load_vocab: special tokens definition check successful ( 323/32064 ). |
|
llm_load_print_meta: format = GGUF V3 (latest) |
|
llm_load_print_meta: arch = llama |
|
llm_load_print_meta: vocab type = SPM |
|
llm_load_print_meta: n_vocab = 32064 |
|
llm_load_print_meta: n_merges = 0 |
|
llm_load_print_meta: n_ctx_train = 65536 |
|
llm_load_print_meta: n_embd = 6144 |
|
llm_load_print_meta: n_head = 48 |
|
llm_load_print_meta: n_head_kv = 8 |
|
llm_load_print_meta: n_layer = 56 |
|
llm_load_print_meta: n_rot = 128 |
|
llm_load_print_meta: n_embd_head_k = 128 |
|
llm_load_print_meta: n_embd_head_v = 128 |
|
llm_load_print_meta: n_gqa = 6 |
|
llm_load_print_meta: n_embd_k_gqa = 1024 |
|
llm_load_print_meta: n_embd_v_gqa = 1024 |
|
llm_load_print_meta: f_norm_eps = 0.0e+00 |
|
llm_load_print_meta: f_norm_rms_eps = 1.0e-05 |
|
llm_load_print_meta: f_clamp_kqv = 0.0e+00 |
|
llm_load_print_meta: f_max_alibi_bias = 0.0e+00 |
|
llm_load_print_meta: f_logit_scale = 0.0e+00 |
|
llm_load_print_meta: n_ff = 16384 |
|
llm_load_print_meta: n_expert = 8 |
|
llm_load_print_meta: n_expert_used = 2 |
|
llm_load_print_meta: causal attn = 1 |
|
llm_load_print_meta: pooling type = 0 |
|
llm_load_print_meta: rope type = 0 |
|
llm_load_print_meta: rope scaling = linear |
|
llm_load_print_meta: freq_base_train = 1000000.0 |
|
llm_load_print_meta: freq_scale_train = 1 |
|
llm_load_print_meta: n_yarn_orig_ctx = 65536 |
|
llm_load_print_meta: rope_finetuned = unknown |
|
llm_load_print_meta: ssm_d_conv = 0 |
|
llm_load_print_meta: ssm_d_inner = 0 |
|
llm_load_print_meta: ssm_d_state = 0 |
|
llm_load_print_meta: ssm_dt_rank = 0 |
|
llm_load_print_meta: model type = 8x22B |
|
llm_load_print_meta: model ftype = F16 |
|
llm_load_print_meta: model params = 140.62 B |
|
llm_load_print_meta: model size = 261.93 GiB (16.00 BPW) |
|
llm_load_print_meta: general.name = internlm2-math-plus-mixtral8x22b |
|
llm_load_print_meta: BOS token = 1 '<s>' |
|
llm_load_print_meta: EOS token = 2 '</s>' |
|
llm_load_print_meta: UNK token = 0 '<unk>' |
|
llm_load_print_meta: LF token = 13 '<0x0A>' |
|
llm_load_print_meta: EOT token = 32004 '<|im_end|>' |
|
ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no |
|
ggml_cuda_init: CUDA_USE_TENSOR_CORES: yes |
|
ggml_cuda_init: found 1 CUDA devices: |
|
Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9, VMM: yes |
|
llm_load_tensors: ggml ctx size = 0.56 MiB |
|
ggml_backend_cuda_buffer_type_alloc_buffer: allocating 9552.47 MiB on device 0: cudaMalloc failed: out of memory |
|
llama_model_load: error loading model: unable to allocate backend buffer |
|
llama_load_model_from_file: failed to load model |
|
llama_init_from_gpt_params: error: failed to load model 'internlm2-math-plus-mixtral8x22b-IMat-GGUF/internlm2-math-plus-mixtral8x22b.gguf' |
|
main : failed to init |
|
[9]2.9094, |
|
save_imatrix: stored collected data after 10 chunks in internlm2-math-plus-mixtral8x22b-IMat-GGUF/imatrix.dat |
|
|
