GPTQ model commit

config.json

@@ -0,0 +1,47 @@
+{
+    "_name_or_path": "/workspace/process/nondzu_mistral-7b-codealpaca-lora/source",
+    "architectures": [
+        "MistralForCausalLM"
+    ],
+    "bos_token_id": 1,
+    "eos_token_id": 2,
+    "hidden_act": "silu",
+    "hidden_size": 4096,
+    "initializer_range": 0.02,
+    "intermediate_size": 14336,
+    "max_position_embeddings": 32768,
+    "model_type": "mistral",
+    "num_attention_heads": 32,
+    "num_hidden_layers": 32,
+    "num_key_value_heads": 8,
+    "pad_token_id": 0,
+    "pretraining_tp": 1,
+    "quantization_config": {
+        "batch_size": 1,
+        "bits": 8,
+        "block_name_to_quantize": "model.layers",
+        "damp_percent": 0.1,
+        "desc_act": true,
+        "disable_exllama": false,
+        "group_size": 32,
+        "max_input_length": null,
+        "model_seqlen": 4096,
+        "module_name_preceding_first_block": [
+            "model.embed_tokens"
+        ],
+        "pad_token_id": null,
+        "quant_method": "gptq",
+        "sym": true,
+        "tokenizer": null,
+        "true_sequential": true,
+        "use_cuda_fp16": true
+    },
+    "rms_norm_eps": 1e-05,
+    "rope_theta": 10000.0,
+    "sliding_window": 4096,
+    "tie_word_embeddings": false,
+    "torch_dtype": "float16",
+    "transformers_version": "4.34.1",
+    "use_cache": true,
+    "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.35.0.dev0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8604ac151feb3caf394987f1bb2e0deadfa046c201c39f770274adb0637a572
+size 8166320464

quantize_config.json

@@ -0,0 +1,8 @@
+{
+    "bits": 8,
+    "group_size": 32,
+    "damp_percent": 0.01,
+    "desc_act": true,
+    "sym": true,
+    "true_sequential": true
+}

run-humaneval.py

@@ -0,0 +1,56 @@
+import torch
+import torch.distributed as dist
+import torch.multiprocessing as mp
+from transformers import AutoTokenizer, LlamaForCausalLM
+from torch.nn.parallel import DistributedDataParallel as DDP
+from evalplus.data import get_human_eval_plus, write_jsonl
+import os
+from tqdm import tqdm  # import tqdm
+
+def setup(rank, world_size):
+    os.environ['MASTER_ADDR'] = 'localhost'
+    os.environ['MASTER_PORT'] = '12355'
+    dist.init_process_group("gloo", rank=rank, world_size=world_size)
+
+def cleanup():
+    dist.destroy_process_group()
+
+def generate_one_completion(ddp_model, tokenizer, prompt: str):
+    tokenizer.pad_token = tokenizer.eos_token
+    inputs = tokenizer(prompt, return_tensors="pt", truncation=True, max_length=4096)
+
+    # Generate
+    generate_ids = ddp_model.module.generate(inputs.input_ids.to("cuda"), max_new_tokens=384, do_sample=True, top_p=0.75, top_k=40, temperature=0.1, pad_token_id=tokenizer.eos_token_id)
+    completion = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+    completion = completion.replace(prompt, "").split("\n\n\n")[0]
+    
+    print("-------------------")
+    print(completion)
+    return completion
+
+def run(rank, world_size):
+    setup(rank, world_size)
+
+    model_path = "Nondzu/Mistral-7B-codealpaca-lora"
+    model = LlamaForCausalLM.from_pretrained(model_path,load_in_8bit=True)
+    ddp_model = DDP(model, device_ids=[rank])
+    tokenizer = AutoTokenizer.from_pretrained(model_path)
+
+    problems = get_human_eval_plus()
+    num_samples_per_task = 1
+
+    samples = [
+        dict(task_id=task_id, completion=generate_one_completion(ddp_model, tokenizer, problems[task_id]["prompt"]))
+        for task_id in tqdm(problems)  # add tqdm here
+        for _ in range(num_samples_per_task)
+    ]
+    write_jsonl(f"samples-Nondzu-Mistral-7B-codealpaca-lora-rank{rank}.jsonl", samples)
+
+    cleanup()
+    
+def main():
+    world_size = 1
+    mp.spawn(run, args=(world_size,), nprocs=world_size, join=True)
+
+if __name__=="__main__":
+    main()

special_tokens_map.json

@@ -0,0 +1,11 @@
+{
+  "additional_special_tokens": [
+    "<unk>",
+    "<s>",
+    "</s>"
+  ],
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "</s>",
+  "unk_token": "<unk>"
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,50 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<unk>",
+    "<s>",
+    "</s>"
+  ],
+  "bos_token": "<s>",
+  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token + ' ' }}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "tokenizer_file": "/home/kamil/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.1/snapshots/7ad5799710574ba1c1d953eba3077af582f3a773/tokenizer.json",
+  "trust_remote_code": false,
+  "unk_token": "<unk>",
+  "use_default_system_prompt": true,
+  "use_fast": true
+}