Training in progress, epoch 1

README.md

@@ -1,10 +1,9 @@
 ---
-license: apache-2.0
-base_model: cyberagent/calm2-7b-chat
+base_model: NousResearch/Llama-2-7b-hf
 tags:
 - generated_from_trainer
 model-index:
-- name: top200mystery
+- name: out
   results: []
 ---
 
@@ -12,9 +11,11 @@ model-index:
 should probably proofread and complete it, then remove this comment. -->
 
 [<img src="https://raw.githubusercontent.com/OpenAccess-AI-Collective/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/OpenAccess-AI-Collective/axolotl)
-# top200mystery
+# out
 
-This model is a fine-tuned version of [cyberagent/calm2-7b-chat](https://huggingface.co/cyberagent/calm2-7b-chat) on an unknown dataset.
+This model is a fine-tuned version of [NousResearch/Llama-2-7b-hf](https://huggingface.co/NousResearch/Llama-2-7b-hf) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.9443
 
 ## Model description
 
@@ -34,18 +35,41 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 0.0002
-- train_batch_size: 32
-- eval_batch_size: 32
+- train_batch_size: 1
+- eval_batch_size: 1
 - seed: 42
 - distributed_type: multi-GPU
 - num_devices: 3
-- gradient_accumulation_steps: 64
-- total_train_batch_size: 6144
-- total_eval_batch_size: 96
+- total_train_batch_size: 3
+- total_eval_batch_size: 3
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
-- lr_scheduler_warmup_steps: 10
-- num_epochs: 3
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 1.0254        | 0.03  | 1    | 3.0959          |
+| 3.2648        | 0.06  | 2    | 3.0959          |
+| 3.0345        | 0.12  | 4    | 1.6018          |
+| 1.4912        | 0.18  | 6    | 1.4104          |
+| 1.4298        | 0.24  | 8    | 1.2483          |
+| 1.2217        | 0.29  | 10   | 1.1785          |
+| 1.1975        | 0.35  | 12   | 1.1200          |
+| 1.1377        | 0.41  | 14   | 1.0922          |
+| 1.0991        | 0.47  | 16   | 1.0625          |
+| 0.9783        | 0.53  | 18   | 1.0422          |
+| 1.0558        | 0.59  | 20   | 1.0100          |
+| 0.9894        | 0.65  | 22   | 0.9902          |
+| 0.9677        | 0.71  | 24   | 0.9780          |
+| 0.9782        | 0.76  | 26   | 0.9679          |
+| 0.9944        | 0.82  | 28   | 0.9595          |
+| 0.9245        | 0.88  | 30   | 0.9509          |
+| 0.9676        | 0.94  | 32   | 0.9468          |
+| 1.0653        | 1.0   | 34   | 0.9443          |
+
 
 ### Framework versions
 

config.json

@@ -17,18 +17,6 @@
   "num_key_value_heads": 32,
   "pad_token_id": 1,
   "pretraining_tp": 1,
-  "quantization_config": {
-    "bnb_4bit_compute_dtype": "bfloat16",
-    "bnb_4bit_quant_type": "nf4",
-    "bnb_4bit_use_double_quant": true,
-    "llm_int8_enable_fp32_cpu_offload": false,
-    "llm_int8_has_fp16_weight": false,
-    "llm_int8_skip_modules": null,
-    "llm_int8_threshold": 6.0,
-    "load_in_4bit": true,
-    "load_in_8bit": false,
-    "quant_method": "bitsandbytes"
-  },
   "rms_norm_eps": 1e-06,
   "rope_scaling": null,
   "rope_theta": 500000,

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_id": 1,
+  "transformers_version": "4.34.1"
+}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:73461114e57467a1db6adb5b561c757096d4059b7cc91137f417aa210b330b1d
-size 4539
+oid sha256:765d7bd291e5773599dd1a122b775bfa93cc0dec79797d7accc64010c9d9dd0d
+size 6011