Training in progress, epoch 3

README.md

@@ -1,9 +1,8 @@
 ---
 license: apache-2.0
-library_name: peft
+base_model: AI-Sweden-Models/gpt-sw3-1.3b
 tags:
 - generated_from_trainer
-base_model: AI-Sweden-Models/gpt-sw3-1.3b
 model-index:
 - name: gpt_icesum
   results: []
@@ -16,7 +15,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [AI-Sweden-Models/gpt-sw3-1.3b](https://huggingface.co/AI-Sweden-Models/gpt-sw3-1.3b) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.7838
+- Loss: 1.7960
 
 ## Model description
 
@@ -35,7 +34,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 5e-05
+- learning_rate: 1e-05
 - train_batch_size: 4
 - eval_batch_size: 4
 - seed: 42
@@ -47,21 +46,20 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 1.9006        | 0.22  | 50   | 1.8021          |
-| 1.907         | 0.44  | 100  | 1.7894          |
-| 1.815         | 0.67  | 150  | 1.7845          |
-| 2.0118        | 0.89  | 200  | 1.7850          |
-| 1.7555        | 1.11  | 250  | 1.7863          |
-| 1.8844        | 1.33  | 300  | 1.7857          |
-| 1.7689        | 1.56  | 350  | 1.7851          |
-| 1.7703        | 1.78  | 400  | 1.7838          |
-| 1.8758        | 2.0   | 450  | 1.7838          |
+| 1.9148        | 0.22  | 50   | 1.8159          |
+| 1.9237        | 0.44  | 100  | 1.8051          |
+| 1.8317        | 0.67  | 150  | 1.8006          |
+| 2.0264        | 0.89  | 200  | 1.7985          |
+| 1.81          | 1.11  | 250  | 1.7961          |
+| 1.9393        | 1.33  | 300  | 1.7951          |
+| 1.8159        | 1.56  | 350  | 1.7934          |
+| 1.8204        | 1.78  | 400  | 1.7959          |
+| 1.9231        | 2.0   | 450  | 1.7960          |
 
 
 ### Framework versions
 
-- PEFT 0.8.2
 - Transformers 4.38.1
 - Pytorch 2.2.0+cu118
 - Datasets 2.17.1
-- Tokenizers 0.15.2
+- Tokenizers 0.15.2

adapter_config.json

@@ -9,13 +9,13 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 512,
+  "lora_alpha": 1024,
   "lora_dropout": 0.1,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 512,
+  "r": 1024,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5191f86f73f75eaadefb7b52bef891a798b837d288cadaa99f589348c1e599ec
-size 201333040
+oid sha256:aea17e760ccaa177a4552a0a7e843a004d2cd263e24ceabd7aa70cc8934a548a
+size 402659704

config.json

@@ -0,0 +1,37 @@
+{
+  "_name_or_path": "AI-Sweden-Models/gpt-sw3-1.3b",
+  "activation_function": "gelu",
+  "apply_query_key_layer_scaling": true,
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 2,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 3,
+  "initializer_range": 0.01,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 2048,
+  "n_embd": 2048,
+  "n_head": 32,
+  "n_inner": 8192,
+  "n_layer": 24,
+  "n_positions": 2048,
+  "normalize_attention_scores": true,
+  "pad_token_id": 0,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "tokenizer_class": "GPTSw3Tokenizer",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.38.1",
+  "use_cache": false,
+  "vocab_size": 64000
+}

generation_config.json

@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": 3,
+  "pad_token_id": 0,
+  "transformers_version": "4.38.1",
+  "use_cache": false
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8974be470606030125c39d48bc73e65ae112771a3d00d092853186840549b159
+size 2889101840

runs/Mar25_15-40-59_gpu-4/events.out.tfevents.1711381259.gpu-4.353385.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0dcca76995e984623df7398a38a1e7add99b20bd780fa12061ab6b0ff1a8bca
+size 26546

runs/Mar25_15-51-22_gpu-4/events.out.tfevents.1711381883.gpu-4.353758.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed307a5f731e995e650bdfa1c9227b569e1e1364cb6b4e5f1bb7dbf99b96a1c4
+size 37125

runs/Mar25_16-20-40_gpu-4/events.out.tfevents.1711383641.gpu-4.354359.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9de70860d9f31fd20d1d76380dfd389e604666b897bba568e0a77e3cce58cdd7
+size 37125

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:35426744f7e2ea623fe429a0fdfb12d139932b3f2bf8bc8cb260147ec97a0cc0
+oid sha256:19f6e5844e102fa35f763b9a71f2d6fbe159f5876ebaa9c1917a40b59c5307f1
 size 4920