usholanb commited on
Commit
62869df
1 Parent(s): 728b15e

Model save

Browse files
README.md CHANGED
@@ -15,7 +15,7 @@ should probably proofread and complete it, then remove this comment. -->
15
 
16
  This model is a fine-tuned version of [HuggingFaceH4/zephyr-7b-beta](https://huggingface.co/HuggingFaceH4/zephyr-7b-beta) on an unknown dataset.
17
  It achieves the following results on the evaluation set:
18
- - Loss: 1.9825
19
 
20
  ## Model description
21
 
@@ -45,13 +45,14 @@ The following hyperparameters were used during training:
45
  - total_eval_batch_size: 4
46
  - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
47
  - lr_scheduler_type: cosine
48
- - num_epochs: 1
49
 
50
  ### Training results
51
 
52
  | Training Loss | Epoch | Step | Validation Loss |
53
  |:-------------:|:-----:|:----:|:---------------:|
54
- | 2.0382 | 0.5 | 1 | 1.9825 |
 
55
 
56
 
57
  ### Framework versions
 
15
 
16
  This model is a fine-tuned version of [HuggingFaceH4/zephyr-7b-beta](https://huggingface.co/HuggingFaceH4/zephyr-7b-beta) on an unknown dataset.
17
  It achieves the following results on the evaluation set:
18
+ - Loss: 1.9786
19
 
20
  ## Model description
21
 
 
45
  - total_eval_batch_size: 4
46
  - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
47
  - lr_scheduler_type: cosine
48
+ - num_epochs: 3.0
49
 
50
  ### Training results
51
 
52
  | Training Loss | Epoch | Step | Validation Loss |
53
  |:-------------:|:-----:|:----:|:---------------:|
54
+ | 2.0382 | 0.5 | 1 | 1.9826 |
55
+ | 2.0382 | 1.65 | 3 | 1.9786 |
56
 
57
 
58
  ### Framework versions
adapter_config.json CHANGED
@@ -16,10 +16,10 @@
16
  "rank_pattern": {},
17
  "revision": null,
18
  "target_modules": [
 
19
  "v_proj",
20
- "q_proj",
21
  "k_proj",
22
- "o_proj"
23
  ],
24
  "task_type": "CAUSAL_LM"
25
  }
 
16
  "rank_pattern": {},
17
  "revision": null,
18
  "target_modules": [
19
+ "o_proj",
20
  "v_proj",
 
21
  "k_proj",
22
+ "q_proj"
23
  ],
24
  "task_type": "CAUSAL_LM"
25
  }
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b3713ad42c5e1ba21cebfac3e251a7609e2638d2e9e85969e2b45443e903d22e
3
  size 109086672
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8c4e020c4d08fc607533170d8afafaf3c335fd06e4e9156b397f69699f04d60f
3
  size 109086672
all_results.json CHANGED
@@ -1,13 +1,13 @@
1
  {
2
- "epoch": 0.5,
3
- "eval_loss": 1.982534408569336,
4
- "eval_runtime": 660.4195,
5
  "eval_samples": 1086,
6
- "eval_samples_per_second": 1.644,
7
  "eval_steps_per_second": 0.412,
8
- "train_loss": 2.0382144451141357,
9
- "train_runtime": 2035.9442,
10
  "train_samples": 2026,
11
- "train_samples_per_second": 0.995,
12
- "train_steps_per_second": 0.0
13
  }
 
1
  {
2
+ "epoch": 1.65,
3
+ "eval_loss": 1.9785882234573364,
4
+ "eval_runtime": 660.2678,
5
  "eval_samples": 1086,
6
+ "eval_samples_per_second": 1.645,
7
  "eval_steps_per_second": 0.412,
8
+ "train_loss": 2.036311705907186,
9
+ "train_runtime": 5409.4181,
10
  "train_samples": 2026,
11
+ "train_samples_per_second": 1.124,
12
+ "train_steps_per_second": 0.001
13
  }
config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "HuggingFaceH4/zephyr-7b-beta",
3
+ "architectures": [
4
+ "MistralForCausalLM"
5
+ ],
6
+ "bos_token_id": 1,
7
+ "eos_token_id": 2,
8
+ "hidden_act": "silu",
9
+ "hidden_size": 4096,
10
+ "initializer_range": 0.02,
11
+ "intermediate_size": 14336,
12
+ "max_position_embeddings": 32768,
13
+ "model_type": "mistral",
14
+ "num_attention_heads": 32,
15
+ "num_hidden_layers": 32,
16
+ "num_key_value_heads": 8,
17
+ "pad_token_id": 2,
18
+ "rms_norm_eps": 1e-05,
19
+ "rope_theta": 10000.0,
20
+ "sliding_window": 4096,
21
+ "tie_word_embeddings": false,
22
+ "torch_dtype": "bfloat16",
23
+ "transformers_version": "4.35.0",
24
+ "use_cache": true,
25
+ "vocab_size": 32000
26
+ }
eval_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
- "epoch": 0.5,
3
- "eval_loss": 1.982534408569336,
4
- "eval_runtime": 660.4195,
5
  "eval_samples": 1086,
6
- "eval_samples_per_second": 1.644,
7
  "eval_steps_per_second": 0.412
8
  }
 
1
  {
2
+ "epoch": 1.65,
3
+ "eval_loss": 1.9785882234573364,
4
+ "eval_runtime": 660.2678,
5
  "eval_samples": 1086,
6
+ "eval_samples_per_second": 1.645,
7
  "eval_steps_per_second": 0.412
8
  }
runs/Dec07_12-10-12_ip-172-16-110-193.ec2.internal/events.out.tfevents.1701951037.ip-172-16-110-193.ec2.internal.5300.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a22926dc2e5f9cf757f1385c526eb6f7c5771243dd5d607a5bc7fdb0ffd548a2
3
+ size 5361
runs/Dec07_12-10-12_ip-172-16-110-193.ec2.internal/events.out.tfevents.1701957107.ip-172-16-110-193.ec2.internal.5300.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6bbb22db19a2827141fecb4f6d38e76d95450f5089fc8656e0114da30665a129
3
+ size 354
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
- "epoch": 0.5,
3
- "train_loss": 2.0382144451141357,
4
- "train_runtime": 2035.9442,
5
  "train_samples": 2026,
6
- "train_samples_per_second": 0.995,
7
- "train_steps_per_second": 0.0
8
  }
 
1
  {
2
+ "epoch": 1.65,
3
+ "train_loss": 2.036311705907186,
4
+ "train_runtime": 5409.4181,
5
  "train_samples": 2026,
6
+ "train_samples_per_second": 1.124,
7
+ "train_steps_per_second": 0.001
8
  }
trainer_state.json CHANGED
@@ -1,42 +1,50 @@
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
- "epoch": 0.504930966469428,
5
  "eval_steps": 500,
6
- "global_step": 1,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.5,
13
- "learning_rate": 0.0,
14
  "loss": 2.0382,
15
  "step": 1
16
  },
17
  {
18
  "epoch": 0.5,
19
- "eval_loss": 1.9825345277786255,
20
- "eval_runtime": 661.3124,
21
- "eval_samples_per_second": 1.642,
22
- "eval_steps_per_second": 0.411,
23
  "step": 1
24
  },
25
  {
26
- "epoch": 0.5,
27
- "step": 1,
28
- "total_flos": 381186736128000.0,
29
- "train_loss": 2.0382144451141357,
30
- "train_runtime": 2035.9442,
31
- "train_samples_per_second": 0.995,
32
- "train_steps_per_second": 0.0
 
 
 
 
 
 
 
 
33
  }
34
  ],
35
  "logging_steps": 5,
36
- "max_steps": 1,
37
- "num_train_epochs": 1,
38
  "save_steps": 500,
39
- "total_flos": 381186736128000.0,
40
  "trial_name": null,
41
  "trial_params": null
42
  }
 
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
+ "epoch": 1.650887573964497,
5
  "eval_steps": 500,
6
+ "global_step": 3,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.5,
13
+ "learning_rate": 1.5000000000000002e-05,
14
  "loss": 2.0382,
15
  "step": 1
16
  },
17
  {
18
  "epoch": 0.5,
19
+ "eval_loss": 1.9825799465179443,
20
+ "eval_runtime": 660.1781,
21
+ "eval_samples_per_second": 1.645,
22
+ "eval_steps_per_second": 0.412,
23
  "step": 1
24
  },
25
  {
26
+ "epoch": 1.65,
27
+ "eval_loss": 1.9785882234573364,
28
+ "eval_runtime": 659.635,
29
+ "eval_samples_per_second": 1.646,
30
+ "eval_steps_per_second": 0.412,
31
+ "step": 3
32
+ },
33
+ {
34
+ "epoch": 1.65,
35
+ "step": 3,
36
+ "total_flos": 1143998697701376.0,
37
+ "train_loss": 2.036311705907186,
38
+ "train_runtime": 5409.4181,
39
+ "train_samples_per_second": 1.124,
40
+ "train_steps_per_second": 0.001
41
  }
42
  ],
43
  "logging_steps": 5,
44
+ "max_steps": 3,
45
+ "num_train_epochs": 3,
46
  "save_steps": 500,
47
+ "total_flos": 1143998697701376.0,
48
  "trial_name": null,
49
  "trial_params": null
50
  }
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:75506c61238dccb8530ced9278c8d1407273ee20b762c69125c93cd0b8ec0e91
3
  size 5688
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:84702a434798da0a39a475c0eb2fd055a2671fb3b9bbd5e70047b1db25e6597c
3
  size 5688