Model save

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+code/wandb/run-20240509_173156-a805570edd5eddcfb7deb3a583d7b4c2/run-a805570edd5eddcfb7deb3a583d7b4c2.wandb filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,63 @@
+---
+license: apache-2.0
+base_model: EleutherAI/pythia-410m-deduped
+tags:
+- generated_from_trainer
+model-index:
+- name: pythia410m-sft-tldr
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# pythia410m-sft-tldr
+
+This model is a fine-tuned version of [EleutherAI/pythia-410m-deduped](https://huggingface.co/EleutherAI/pythia-410m-deduped) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 2.5805
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 3e-06
+- train_batch_size: 32
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 128
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- num_epochs: 1.0
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 2.7228        | 0.2   | 183  | 2.6403          |
+| 2.6244        | 0.4   | 366  | 2.6018          |
+| 2.5986        | 0.6   | 549  | 2.5865          |
+| 2.5808        | 0.8   | 732  | 2.5805          |
+
+
+### Framework versions
+
+- Transformers 4.38.2
+- Pytorch 2.1.2+cu121
+- Datasets 2.17.0
+- Tokenizers 0.15.2

code/wandb/run-20240509_173156-a805570edd5eddcfb7deb3a583d7b4c2/files/config.yaml

@@ -81,6 +81,26 @@ _wandb:
       5: 1
       6:
       - 1
+    - 1: train/train_runtime
+      5: 1
+      6:
+      - 1
+    - 1: train/train_samples_per_second
+      5: 1
+      6:
+      - 1
+    - 1: train/train_steps_per_second
+      5: 1
+      6:
+      - 1
+    - 1: train/total_flos
+      5: 1
+      6:
+      - 1
+    - 1: train/train_loss
+      5: 1
+      6:
+      - 1
 return_dict:
   desc: null
   value: true

code/wandb/run-20240509_173156-a805570edd5eddcfb7deb3a583d7b4c2/files/output.log

@@ -580,3 +580,482 @@
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 60%|██████    | 549/912 [25:58<15:40,  2.59s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 98%|█████████▊| 793/806 [00:57<00:00, 13.76it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 66%|██████▌   | 599/912 [29:06<13:31,  2.59s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 77%|███████▋  | 699/912 [33:26<09:12,  2.59s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 80%|████████  | 732/912 [34:51<07:50,  2.61s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 98%|█████████▊| 793/806 [00:57<00:00, 13.76it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 88%|████████▊ | 799/912 [38:42<04:55,  2.61s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 99%|█████████▊| 899/912 [43:00<00:33,  2.55s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+100%|█████████▉| 911/912 [43:31<00:02,  2.55s/it]
+
+100%|██████████| 912/912 [43:33<00:00,  2.87s/it]
+run-a805570edd5eddcfb7deb3a583d7b4c2.wandb: 100%|██████████| 1.45M/1.45M [00:00<00:00, 6.26MB/s]
+Upload 2 LFS files:  50%|█████     | 1/2 [00:00<00:00,  1.91it/s]05kB/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

code/wandb/run-20240509_173156-a805570edd5eddcfb7deb3a583d7b4c2/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 2.5986, "train/grad_norm": 5.898989677429199, "train/learning_rate": 1.2735173894411445e-06, "train/epoch": 0.55, "train/global_step": 500, "_timestamp": 1715277332.9971147, "_runtime": 1416.2355046272278, "_step": 6, "eval/loss": 2.601827383041382, "eval/runtime": 58.2033, "eval/samples_per_second": 110.767, "eval/steps_per_second": 13.848}
+{"train/loss": 2.5779, "train/grad_norm": 3.9158174991607666, "train/learning_rate": 1.2813624190484708e-09, "train/epoch": 1.0, "train/global_step": 912, "_timestamp": 1715278532.1392817, "_runtime": 2615.3776717185974, "_step": 13, "eval/loss": 2.5804877281188965, "eval/runtime": 58.1953, "eval/samples_per_second": 110.782, "eval/steps_per_second": 13.85, "train/train_runtime": 2617.6291, "train/train_samples_per_second": 44.591, "train/train_steps_per_second": 0.348, "train/total_flos": 1.3292726803182387e+17, "train/train_loss": 2.614081516600492}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "transformers_version": "4.38.2"
+}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:39a41ac637b2dfbe095630e36ca10e98d887b503fc8ccf23318b7be0ad4db577
+oid sha256:c55280af0a3145842985599424628918297ee98f6d663db70b42df86aa2de775
 size 1621370224