Upload model

README.md

@@ -5,7 +5,7 @@ tags:
 - trl
 - sft
 - generated_from_trainer
-base_model: bigcode/starcoder2-3b
+base_model: bigcode/starcoder2-15b
 model-index:
 - name: finetune_starcoder2
   results: []
@@ -14,10 +14,10 @@ model-index:
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/vall-e/huggingface/runs/c40h1xmy)
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/vall-e/huggingface/runs/pflx7g43)
 # finetune_starcoder2
 
-This model is a fine-tuned version of [bigcode/starcoder2-3b](https://huggingface.co/bigcode/starcoder2-3b) on an unknown dataset.
+This model is a fine-tuned version of [bigcode/starcoder2-15b](https://huggingface.co/bigcode/starcoder2-15b) on an unknown dataset.
 
 ## Model description
 
@@ -54,8 +54,8 @@ The following hyperparameters were used during training:
 
 ### Framework versions
 
-- PEFT 0.10.0
-- Transformers 4.41.0.dev0
-- Pytorch 2.2.1+cu121
+- PEFT 0.11.1
+- Transformers 4.42.0.dev0
+- Pytorch 2.3.0+cu121
 - Datasets 2.19.1
 - Tokenizers 0.19.1

adapter_config.json

@@ -1,7 +1,7 @@
 {
   "alpha_pattern": {},
   "auto_mapping": null,
-  "base_model_name_or_path": "bigcode/starcoder2-3b",
+  "base_model_name_or_path": "bigcode/starcoder2-15b",
   "bias": "none",
   "fan_in_fan_out": false,
   "inference_mode": true,
@@ -20,12 +20,12 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "gate_proj",
-    "o_proj",
-    "q_proj",
+    "k_proj",
     "v_proj",
+    "o_proj",
     "up_proj",
-    "k_proj",
+    "gate_proj",
+    "q_proj",
     "down_proj"
   ],
   "task_type": "CAUSAL_LM",

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:236c9e6a2f7667afc5e4fe1fa8301badae50d847dd2d08f4e036b497173aeefb
-size 18218008
+oid sha256:9287a32a3396d0bd4d372e0c90796c32761ed4f7c2a9b59560f8f32e306b6fbf
+size 48539256

final_checkpoint/config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "bigcode/starcoder2-3b",
+  "_name_or_path": "bigcode/starcoder2-15b",
   "architectures": [
     "Starcoder2ForCausalLM"
   ],
@@ -8,17 +8,17 @@
   "embedding_dropout": 0.1,
   "eos_token_id": 0,
   "hidden_act": "gelu_pytorch_tanh",
-  "hidden_size": 3072,
-  "initializer_range": 0.018042,
-  "intermediate_size": 12288,
+  "hidden_size": 6144,
+  "initializer_range": 0.01275,
+  "intermediate_size": 24576,
   "max_position_embeddings": 16384,
   "mlp_type": "default",
   "model_type": "starcoder2",
   "norm_epsilon": 1e-05,
   "norm_type": "layer_norm",
-  "num_attention_heads": 24,
-  "num_hidden_layers": 30,
-  "num_key_value_heads": 2,
+  "num_attention_heads": 48,
+  "num_hidden_layers": 40,
+  "num_key_value_heads": 4,
   "quantization_config": {
     "_load_in_4bit": true,
     "_load_in_8bit": false,
@@ -35,10 +35,11 @@
     "quant_method": "bitsandbytes"
   },
   "residual_dropout": 0.1,
-  "rope_theta": 999999.4420358813,
+  "rope_theta": 100000,
   "sliding_window": 4096,
+  "tie_word_embeddings": false,
   "torch_dtype": "float32",
-  "transformers_version": "4.41.0.dev0",
+  "transformers_version": "4.42.0.dev0",
   "use_bias": true,
   "use_cache": true,
   "vocab_size": 49152

final_checkpoint/generation_config.json

@@ -1,6 +1,6 @@
 {
   "_from_model_config": true,
-  "bos_token_id": 0,
-  "eos_token_id": 0,
-  "transformers_version": "4.41.0.dev0"
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.42.0.dev0"
 }

final_checkpoint/model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50f444c7f2952ebfc56ace73cce35b9422ca6c2deeac45776a897e571ff9e7de
+size 4950280112

final_checkpoint/model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81726e7af63cab3a8e8a7ec6d35e3216b7e38b19074bd121beaaab6014a3eee6
+size 4951978560

final_checkpoint/model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8cf600df2701898434e15f495df429d54421618ab52daf150ca73b519223f0e
+size 1207959680

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cd99eeb1354dd187a1e8d318e97524ff7581dd4ddcb0270699b827f045376365
+oid sha256:237fa6df610231b92c1bf8704ec441693524b8f0958486b43eb6dd5d2c63340c
 size 5112