Upload folder using huggingface_hub

README.md

@@ -1,32 +1,21 @@
 ---
-license: mit
-language:
-- ar
+library_name: peft
 ---
-TigerBot-7B LAPT + CLP+ Arabic
-===
+## Training procedure
 
-## How to use
-```python
-from peft import AutoPeftModelForCausalLM
-from transformers import AutoTokenizer
 
-model = AutoPeftModelForCausalLM.from_pretrained(
-  "atsuki-yamaguchi/tigerbot-7b-base-clpp-ar"
-)
-```
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+### Framework versions
 
-## Citation
-```
-@article{yamaguchi2024empirical,
-  title={An Empirical Study on Cross-lingual Vocabulary Adaptation for Efficient Generative {LLM} Inference}, 
-  author={Atsuki Yamaguchi and Aline Villavicencio and Nikolaos Aletras},
-  journal={ArXiv},
-  year={2024},
-  volume={abs/2402.10712},
-  url={https://arxiv.org/abs/2402.10712}
-}
-```
 
-## Link
-For more details, please visit https://github.com/gucci-j/llm-cva
+- PEFT 0.5.0

adapter_config.json

@@ -1,29 +1 @@
-{
-  "auto_mapping": null,
-  "base_model_name_or_path": "atsuki-yamaguchi/tigerbot-7b-base-clpp-ar",
-  "bias": "none",
-  "fan_in_fan_out": false,
-  "inference_mode": true,
-  "init_lora_weights": true,
-  "layers_pattern": null,
-  "layers_to_transform": null,
-  "lora_alpha": 32,
-  "lora_dropout": 0.05,
-  "modules_to_save": [
-    "lm_head",
-    "embed_tokens"
-  ],
-  "peft_type": "LORA",
-  "r": 8,
-  "revision": null,
-  "target_modules": [
-    "q_proj",
-    "v_proj",
-    "k_proj",
-    "o_proj",
-    "gate_proj",
-    "down_proj",
-    "up_proj"
-  ],
-  "task_type": "CAUSAL_LM"
-}
+{"auto_mapping": null, "base_model_name_or_path": "atsuki-yamaguchi/tigerbot-7b-base-clpp-ar", "bias": "none", "fan_in_fan_out": false, "inference_mode": true, "init_lora_weights": true, "layers_pattern": null, "layers_to_transform": null, "lora_alpha": 32, "lora_dropout": 0.05, "modules_to_save": ["lm_head", "embed_tokens"], "peft_type": "LORA", "r": 8, "revision": null, "target_modules": ["q_proj", "v_proj", "k_proj", "o_proj", "gate_proj", "down_proj", "up_proj"], "task_type": "CAUSAL_LM"}

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "/mnt/parscratch/users/acp23ay/private/models/tigerbot-7b-base-ar-clp-plus",
+  "_name_or_path": "TigerResearch/tigerbot-7b-base",
   "architectures": [
     "LlamaForCausalLM"
   ],
@@ -21,7 +21,7 @@
   "rope_scaling": null,
   "rope_theta": 10000.0,
   "tie_word_embeddings": false,
-  "torch_dtype": "float32",
+  "torch_dtype": "float64",
   "transformers_version": "4.35.0.dev0",
   "use_cache": true,
   "vocab_size": 64000

model-00001-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cf172036d5beb5eba82b40f8326418445035ed0048711800269d46df03b33245
-size 4915860320
+oid sha256:55f91776d14d120e90959697b31448ce6b94f6f7c86abbd7602e95e4cd2e2605
+size 4974546664

model-00002-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c0caf7138553ae36aa80702d1218a23873098a5e73cd11e25a6caf7080071817
-size 4857206856
+oid sha256:997fb2ef0df37cba255743a67d2e124712010dc421564d652de9bca9a13ac77b
+size 4857206848

model-00003-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9f7006a6f327bc54c4ff7b414274088e35824b98bb7ed225129d766ff4c5af01
+oid sha256:63f7f1ad4ce7ef01c8e1f770d4992dab3683b53f480ea7e16e2d2cd131ba9ae4
 size 4857206896

model-00004-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d5d3bd310eaa968845955c0521aa3f34822cb9a8625ed12d502fd322321dc52f
+oid sha256:70d5d8b33baba96365bb16b3c8191e177de16c2ac7da20f49b48d859ecc3fb34
 size 4857206896

model-00005-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c17f81881a4fdeead802ed7936c8099f8c53bf191816727c50271b83818af749
+oid sha256:50893eaddd16b7542245ac6d814e07bea7357fc37b3bdc70398b9c1f872d935e
 size 4857206896

model-00006-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d30e54a85473acc143d4180d57068b4587f0bab9108f7012253839f097227bf5
-size 3657584232
+oid sha256:78d3753464469fb030b6693ae8d85b173c5439fb8e482e937798cde0b2aa21e7
+size 4647473904

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 28002238464
+    "total_size": 29050814464
   },
   "weight_map": {
     "lm_head.weight": "model-00006-of-00006.safetensors",
@@ -25,13 +25,13 @@
     "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
     "model.layers.10.input_layernorm.weight": "model-00003-of-00006.safetensors",
     "model.layers.10.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
-    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
-    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
     "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
-    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
-    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
-    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
-    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
     "model.layers.11.input_layernorm.weight": "model-00003-of-00006.safetensors",
     "model.layers.11.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
     "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
@@ -68,24 +68,24 @@
     "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
     "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
     "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
-    "model.layers.15.input_layernorm.weight": "model-00003-of-00006.safetensors",
-    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
     "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
-    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
-    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
     "model.layers.15.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
     "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
     "model.layers.15.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
     "model.layers.15.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
     "model.layers.16.input_layernorm.weight": "model-00004-of-00006.safetensors",
     "model.layers.16.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
-    "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
-    "model.layers.16.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
     "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
-    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
-    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
-    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
-    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
     "model.layers.17.input_layernorm.weight": "model-00004-of-00006.safetensors",
     "model.layers.17.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
     "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
@@ -131,24 +131,24 @@
     "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
     "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
     "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
-    "model.layers.21.input_layernorm.weight": "model-00004-of-00006.safetensors",
-    "model.layers.21.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
     "model.layers.21.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
-    "model.layers.21.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
-    "model.layers.21.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
     "model.layers.21.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
     "model.layers.21.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
     "model.layers.21.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
     "model.layers.21.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
     "model.layers.22.input_layernorm.weight": "model-00005-of-00006.safetensors",
     "model.layers.22.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
-    "model.layers.22.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
-    "model.layers.22.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
     "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
-    "model.layers.22.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
-    "model.layers.22.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
-    "model.layers.22.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
-    "model.layers.22.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
     "model.layers.23.input_layernorm.weight": "model-00005-of-00006.safetensors",
     "model.layers.23.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
     "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
@@ -185,24 +185,24 @@
     "model.layers.26.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
     "model.layers.26.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
     "model.layers.26.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
-    "model.layers.27.input_layernorm.weight": "model-00005-of-00006.safetensors",
-    "model.layers.27.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
     "model.layers.27.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
-    "model.layers.27.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
-    "model.layers.27.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
     "model.layers.27.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
     "model.layers.27.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
     "model.layers.27.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
     "model.layers.27.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
     "model.layers.28.input_layernorm.weight": "model-00006-of-00006.safetensors",
     "model.layers.28.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
-    "model.layers.28.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
-    "model.layers.28.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
     "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
-    "model.layers.28.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
-    "model.layers.28.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
-    "model.layers.28.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
-    "model.layers.28.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
     "model.layers.29.input_layernorm.weight": "model-00006-of-00006.safetensors",
     "model.layers.29.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
     "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
@@ -212,11 +212,11 @@
     "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
     "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
     "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
-    "model.layers.3.input_layernorm.weight": "model-00001-of-00006.safetensors",
-    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
     "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
-    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
-    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
     "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
     "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
     "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
@@ -241,13 +241,13 @@
     "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
     "model.layers.4.input_layernorm.weight": "model-00002-of-00006.safetensors",
     "model.layers.4.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
-    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
-    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
     "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
-    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
-    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
-    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
-    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
     "model.layers.5.input_layernorm.weight": "model-00002-of-00006.safetensors",
     "model.layers.5.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
     "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
@@ -284,11 +284,11 @@
     "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
     "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
     "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
-    "model.layers.9.input_layernorm.weight": "model-00002-of-00006.safetensors",
-    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
     "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
-    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
-    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
     "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
     "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
     "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d3ca824474c990b0e0eed473e27e44726039e49022c1ce41a044b075a7b16e2
+size 1091197148

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf529198e9c26a03f9e5d5faa128e2b28c42f44f340935dece0e18c806cef182
+size 14244

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b21283c3db24950204115ac348f58e53c6d0488c1b3816d199a477f9c0931a6
+size 1064

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a34ccdaf89aac619bed9397b6efe5813df7d048bbaf2f1490e50b6ed605b556b
+size 4664