mogmyij commited on
Commit
31b2647
1 Parent(s): 48817ec

End of training

Browse files
README.md CHANGED
@@ -18,7 +18,7 @@ should probably proofread and complete it, then remove this comment. -->
18
 
19
  This model is a fine-tuned version of [meta-llama/Llama-2-7b-hf](https://huggingface.co/meta-llama/Llama-2-7b-hf) on an unknown dataset.
20
  It achieves the following results on the evaluation set:
21
- - Loss: 0.9204
22
 
23
  ## Model description
24
 
@@ -38,11 +38,11 @@ More information needed
38
 
39
  The following hyperparameters were used during training:
40
  - learning_rate: 0.0001
41
- - train_batch_size: 1
42
  - eval_batch_size: 8
43
  - seed: 42
44
  - gradient_accumulation_steps: 4
45
- - total_train_batch_size: 4
46
  - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
47
  - lr_scheduler_type: cosine
48
  - lr_scheduler_warmup_ratio: 0.05
@@ -51,11 +51,11 @@ The following hyperparameters were used during training:
51
 
52
  ### Training results
53
 
54
- | Training Loss | Epoch | Step | Validation Loss |
55
- |:-------------:|:-----:|:----:|:---------------:|
56
- | 0.4853 | 1.0 | 225 | 0.4799 |
57
- | 0.0102 | 2.0 | 450 | 0.6512 |
58
- | 0.0026 | 3.0 | 675 | 0.9204 |
59
 
60
 
61
  ### Framework versions
 
18
 
19
  This model is a fine-tuned version of [meta-llama/Llama-2-7b-hf](https://huggingface.co/meta-llama/Llama-2-7b-hf) on an unknown dataset.
20
  It achieves the following results on the evaluation set:
21
+ - Loss: 0.5874
22
 
23
  ## Model description
24
 
 
38
 
39
  The following hyperparameters were used during training:
40
  - learning_rate: 0.0001
41
+ - train_batch_size: 2
42
  - eval_batch_size: 8
43
  - seed: 42
44
  - gradient_accumulation_steps: 4
45
+ - total_train_batch_size: 8
46
  - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
47
  - lr_scheduler_type: cosine
48
  - lr_scheduler_warmup_ratio: 0.05
 
51
 
52
  ### Training results
53
 
54
+ | Training Loss | Epoch | Step | Validation Loss |
55
+ |:-------------:|:------:|:----:|:---------------:|
56
+ | 0.5743 | 0.9996 | 1178 | 0.3069 |
57
+ | 0.3457 | 2.0 | 2357 | 0.3704 |
58
+ | 0.9745 | 2.9987 | 3534 | 0.5874 |
59
 
60
 
61
  ### Framework versions
adapter_config.json CHANGED
@@ -10,20 +10,20 @@
10
  "layers_pattern": null,
11
  "layers_to_transform": null,
12
  "loftq_config": {},
13
- "lora_alpha": 8,
14
  "lora_dropout": 0.05,
15
  "megatron_config": null,
16
  "megatron_core": "megatron.core",
17
  "modules_to_save": null,
18
  "peft_type": "LORA",
19
- "r": 4,
20
  "rank_pattern": {},
21
  "revision": null,
22
  "target_modules": [
23
- "o_proj",
24
- "k_proj",
25
  "v_proj",
26
- "q_proj"
 
 
27
  ],
28
  "task_type": "CAUSAL_LM",
29
  "use_dora": false,
 
10
  "layers_pattern": null,
11
  "layers_to_transform": null,
12
  "loftq_config": {},
13
+ "lora_alpha": 16,
14
  "lora_dropout": 0.05,
15
  "megatron_config": null,
16
  "megatron_core": "megatron.core",
17
  "modules_to_save": null,
18
  "peft_type": "LORA",
19
+ "r": 8,
20
  "rank_pattern": {},
21
  "revision": null,
22
  "target_modules": [
 
 
23
  "v_proj",
24
+ "q_proj",
25
+ "k_proj",
26
+ "o_proj"
27
  ],
28
  "task_type": "CAUSAL_LM",
29
  "use_dora": false,
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:f659948fbed3b49803ed7ccdd66e85498a2547dfee7b61062a196765eac5d71d
3
- size 16811144
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e96718103d173e42490b1ceb7c86fce5bf932d3fb6d2e2ecc0462d4eefade8e4
3
+ size 33588528
tokenizer.json CHANGED
@@ -1,11 +1,6 @@
1
  {
2
  "version": "1.0",
3
- "truncation": {
4
- "direction": "Right",
5
- "max_length": 512,
6
- "strategy": "LongestFirst",
7
- "stride": 0
8
- },
9
  "padding": null,
10
  "added_tokens": [
11
  {
 
1
  {
2
  "version": "1.0",
3
+ "truncation": null,
 
 
 
 
 
4
  "padding": null,
5
  "added_tokens": [
6
  {
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:6be1299fa2bf7bae6292b38b856fcd5b24099e3021e64eaf9e2572a4e7ca9ea5
3
  size 5112
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:88c7be8d34d851b21daef72d562e9cd1c59b7dcae7a4bd60cb8344dfdf1c7a41
3
  size 5112