dwb2023/llama38binstruct_summarize_v3

README.md

@@ -20,7 +20,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [NousResearch/Meta-Llama-3-8B-Instruct](https://huggingface.co/NousResearch/Meta-Llama-3-8B-Instruct) on the generator dataset.
 It achieves the following results on the evaluation set:
-- Loss: 2.0847
+- Loss: 1.9957
 
 ## Model description
 
@@ -50,12 +50,12 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch  | Step | Validation Loss |
-|:-------------:|:------:|:----:|:---------------:|
-| 1.4079        | 1.1905 | 25   | 1.4325          |
-| 0.3935        | 2.3810 | 50   | 1.6786          |
-| 0.3836        | 3.5714 | 75   | 1.7694          |
-| 0.1039        | 4.7619 | 100  | 2.0847          |
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 1.4216        | 1.25  | 25   | 1.4108          |
+| 0.5039        | 2.5   | 50   | 1.6955          |
+| 0.183         | 3.75  | 75   | 1.8566          |
+| 0.1127        | 5.0   | 100  | 1.9957          |
 
 
 ### Framework versions
@@ -63,5 +63,5 @@ The following hyperparameters were used during training:
 - PEFT 0.11.1
 - Transformers 4.41.2
 - Pytorch 2.3.0+cu121
-- Datasets 2.19.2
+- Datasets 2.20.0
 - Tokenizers 0.19.1

adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "up_proj",
-    "k_proj",
     "o_proj",
-    "down_proj",
     "v_proj",
+    "q_proj",
+    "down_proj",
     "gate_proj",
-    "q_proj"
+    "k_proj",
+    "up_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d5eb2ba5bd31fe265e089ab86d5ca7f0167c063cc985c997332e6d56c04fb999
-size 167832240
+oid sha256:e44ce263e6fd885f50d82ca515b9325375b43ee36ededb75acf161ce88bc2e41
+size 48

config.json

@@ -6,7 +6,7 @@
   "attention_bias": false,
   "attention_dropout": 0.0,
   "bos_token_id": 128000,
-  "eos_token_id": 128001,
+  "eos_token_id": 128009,
   "hidden_act": "silu",
   "hidden_size": 4096,
   "initializer_range": 0.02,
@@ -18,12 +18,27 @@
   "num_hidden_layers": 32,
   "num_key_value_heads": 8,
   "pretraining_tp": 1,
+  "quantization_config": {
+    "_load_in_4bit": true,
+    "_load_in_8bit": false,
+    "bnb_4bit_compute_dtype": "float16",
+    "bnb_4bit_quant_storage": "uint8",
+    "bnb_4bit_quant_type": "nf4",
+    "bnb_4bit_use_double_quant": true,
+    "llm_int8_enable_fp32_cpu_offload": false,
+    "llm_int8_has_fp16_weight": false,
+    "llm_int8_skip_modules": null,
+    "llm_int8_threshold": 6.0,
+    "load_in_4bit": true,
+    "load_in_8bit": false,
+    "quant_method": "bitsandbytes"
+  },
   "rms_norm_eps": 1e-05,
   "rope_scaling": null,
   "rope_theta": 500000.0,
   "tie_word_embeddings": false,
   "torch_dtype": "float32",
   "transformers_version": "4.41.2",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 128256
 }

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6724f0c2a41ca11368fe260624fa760a1b7ac1d936f76b17ba92b652dc150ec1
+size 4997579690

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5719f4c9810195ddf264a0868adaa25a2802b31c5acce12347a51e6a8dd2d1cd
+size 2807050004

runs/Jun14_05-45-11_2324becbf0e1/events.out.tfevents.1718343913.2324becbf0e1.965.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39b2ad3a770376634e6269c0288bd4673324837d60d18330c6404c2e9ecaf5c2
+size 9238

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:88ec13acf05418ea06f7ef2cb995791d2f2ad262c2272593cce204cea317952b
+oid sha256:ddeed7fc25a7b2d97345ae61c62c8ef1296bab1c3c6a1c366bd84db2d6c05323
 size 5368