commit files to HF hub

README.md

@@ -2,22 +2,35 @@
 license: apache-2.0
 tags:
 - generated_from_trainer
+datasets:
+- eli5
 metrics:
 - accuracy
 model-index:
-- name: jason-expert-eli5-0.5k-same-ds
-  results: []
+- name: layer_9,10,11,12,13
+  results:
+  - task:
+      type: text-generation
+      name: Causal Language Modeling
+    dataset:
+      name: eli5
+      type: eli5
+      split: None
+    metrics:
+    - type: accuracy
+      value: 0.49515362035225047
+      name: Accuracy
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-# jason-expert-eli5-0.5k-same-ds
+# layer_9,10,11,12,13
 
-This model is a fine-tuned version of [EleutherAI/pythia-1b-deduped](https://huggingface.co/EleutherAI/pythia-1b-deduped) on the None dataset.
+This model is a fine-tuned version of [EleutherAI/pythia-1b-deduped](https://huggingface.co/EleutherAI/pythia-1b-deduped) on the eli5 dataset.
 It achieves the following results on the evaluation set:
-- Loss: 2.7820
-- Accuracy: 0.4294
+- Loss: 2.4496
+- Accuracy: 0.4952
 
 ## Model description
 
@@ -63,3 +76,7 @@ The following hyperparameters were used during training:
 - Pytorch 2.0.1+rocm5.4.2
 - Datasets 2.11.0
 - Tokenizers 0.13.3
+
+
+## Wandb Report
+ https://wandb.ai/ontocord/jason-test-pythia-1b-deduped-layer-test-eli5/runs/eecbe79c

all_results.json

@@ -1,9 +1,15 @@
 {
-    "eval_accuracy": 0.06482681017612524,
-    "eval_loss": 5.38467264175415,
-    "eval_runtime": 24.788,
+    "epoch": 14.81,
+    "eval_accuracy": 0.49515362035225047,
+    "eval_loss": 2.449622631072998,
+    "eval_runtime": 20.7527,
     "eval_samples": 2000,
-    "eval_samples_per_second": 80.684,
-    "eval_steps_per_second": 1.291,
-    "perplexity": 218.03871565182433
+    "eval_samples_per_second": 96.373,
+    "eval_steps_per_second": 1.542,
+    "perplexity": 11.58397446228507,
+    "train_loss": 3.714359001159668,
+    "train_runtime": 485.1916,
+    "train_samples": 2153,
+    "train_samples_per_second": 65.953,
+    "train_steps_per_second": 1.031
 }

eval_results.json

@@ -1,9 +1,10 @@
 {
-    "eval_accuracy": 0.06482681017612524,
-    "eval_loss": 5.38467264175415,
-    "eval_runtime": 24.788,
+    "epoch": 14.81,
+    "eval_accuracy": 0.49515362035225047,
+    "eval_loss": 2.449622631072998,
+    "eval_runtime": 20.7527,
     "eval_samples": 2000,
-    "eval_samples_per_second": 80.684,
-    "eval_steps_per_second": 1.291,
-    "perplexity": 218.03871565182433
+    "eval_samples_per_second": 96.373,
+    "eval_steps_per_second": 1.542,
+    "perplexity": 11.58397446228507
 }

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 14.81,
+    "train_loss": 3.714359001159668,
+    "train_runtime": 485.1916,
+    "train_samples": 2153,
+    "train_samples_per_second": 65.953,
+    "train_steps_per_second": 1.031
+}

trainer_state.json

@@ -0,0 +1,193 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 14.814814814814815,
+  "global_step": 500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.59,
+      "learning_rate": 9.6e-05,
+      "loss": 5.0937,
+      "step": 20
+    },
+    {
+      "epoch": 1.19,
+      "learning_rate": 9.200000000000001e-05,
+      "loss": 4.8773,
+      "step": 40
+    },
+    {
+      "epoch": 1.78,
+      "learning_rate": 8.800000000000001e-05,
+      "loss": 4.7701,
+      "step": 60
+    },
+    {
+      "epoch": 2.37,
+      "learning_rate": 8.4e-05,
+      "loss": 4.6992,
+      "step": 80
+    },
+    {
+      "epoch": 2.96,
+      "learning_rate": 8e-05,
+      "loss": 4.6032,
+      "step": 100
+    },
+    {
+      "epoch": 3.56,
+      "learning_rate": 7.6e-05,
+      "loss": 4.5531,
+      "step": 120
+    },
+    {
+      "epoch": 4.15,
+      "learning_rate": 7.2e-05,
+      "loss": 4.5023,
+      "step": 140
+    },
+    {
+      "epoch": 4.74,
+      "learning_rate": 6.800000000000001e-05,
+      "loss": 4.4622,
+      "step": 160
+    },
+    {
+      "epoch": 5.33,
+      "learning_rate": 6.400000000000001e-05,
+      "loss": 4.3641,
+      "step": 180
+    },
+    {
+      "epoch": 5.93,
+      "learning_rate": 6e-05,
+      "loss": 4.3186,
+      "step": 200
+    },
+    {
+      "epoch": 5.93,
+      "eval_accuracy": 0.13991878669275928,
+      "eval_loss": 4.384984493255615,
+      "eval_runtime": 20.6329,
+      "eval_samples_per_second": 96.933,
+      "eval_steps_per_second": 1.551,
+      "step": 200
+    },
+    {
+      "epoch": 6.52,
+      "learning_rate": 5.6000000000000006e-05,
+      "loss": 4.1148,
+      "step": 220
+    },
+    {
+      "epoch": 7.11,
+      "learning_rate": 5.2000000000000004e-05,
+      "loss": 4.0925,
+      "step": 240
+    },
+    {
+      "epoch": 7.7,
+      "learning_rate": 4.8e-05,
+      "loss": 3.8203,
+      "step": 260
+    },
+    {
+      "epoch": 8.3,
+      "learning_rate": 4.4000000000000006e-05,
+      "loss": 3.6945,
+      "step": 280
+    },
+    {
+      "epoch": 8.89,
+      "learning_rate": 4e-05,
+      "loss": 3.5112,
+      "step": 300
+    },
+    {
+      "epoch": 9.48,
+      "learning_rate": 3.6e-05,
+      "loss": 3.3269,
+      "step": 320
+    },
+    {
+      "epoch": 10.07,
+      "learning_rate": 3.2000000000000005e-05,
+      "loss": 3.2166,
+      "step": 340
+    },
+    {
+      "epoch": 10.67,
+      "learning_rate": 2.8000000000000003e-05,
+      "loss": 2.9733,
+      "step": 360
+    },
+    {
+      "epoch": 11.26,
+      "learning_rate": 2.4e-05,
+      "loss": 2.8717,
+      "step": 380
+    },
+    {
+      "epoch": 11.85,
+      "learning_rate": 2e-05,
+      "loss": 2.7653,
+      "step": 400
+    },
+    {
+      "epoch": 11.85,
+      "eval_accuracy": 0.4293894324853229,
+      "eval_loss": 2.7819876670837402,
+      "eval_runtime": 22.9408,
+      "eval_samples_per_second": 87.181,
+      "eval_steps_per_second": 1.395,
+      "step": 400
+    },
+    {
+      "epoch": 12.44,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 2.5936,
+      "step": 420
+    },
+    {
+      "epoch": 13.04,
+      "learning_rate": 1.2e-05,
+      "loss": 2.5338,
+      "step": 440
+    },
+    {
+      "epoch": 13.63,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 2.3836,
+      "step": 460
+    },
+    {
+      "epoch": 14.22,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 2.3762,
+      "step": 480
+    },
+    {
+      "epoch": 14.81,
+      "learning_rate": 0.0,
+      "loss": 2.3408,
+      "step": 500
+    },
+    {
+      "epoch": 14.81,
+      "step": 500,
+      "total_flos": 8.933464866816e+16,
+      "train_loss": 3.714359001159668,
+      "train_runtime": 485.1916,
+      "train_samples_per_second": 65.953,
+      "train_steps_per_second": 1.031
+    }
+  ],
+  "max_steps": 500,
+  "num_train_epochs": 16,
+  "total_flos": 8.933464866816e+16,
+  "trial_name": null,
+  "trial_params": null
+}