Hugging Face's logo

AKTaylor98
/
Llama2-7b-sft-qlora

PEFT
TensorBoard
Safetensors
llama
trl
sft
Generated from Trainer
4-bit precision
bitsandbytes
Model card Files Metrics Community
AKTaylor98 commited on
Commit
a78f671
•
1 Parent(s): 9926325

Model save

Browse files
Files changed (21) hide show
  1. README.md +2 -2
  2. adapter_config.json +4 -4
  3. all_results.json +8 -3
  4. config.json +40 -0
  5. dataset_dict.json +1 -1
  6. eval_results.json +8 -0
  7. evaluation/data-00000-of-00001.arrow +3 -0
  8. evaluation/dataset_info.json +48 -0
  9. evaluation/state.json +13 -0
  10. runs/May23_17-07-55_scai5.cs.ucla.edu/events.out.tfevents.1716509488.scai5.cs.ucla.edu.1145330.1 +3 -0
  11. runs/May23_17-23-15_scai5.cs.ucla.edu/events.out.tfevents.1716510206.scai5.cs.ucla.edu.1148301.0 +3 -0
  12. test/cache-6d56a9de8848869b.arrow +3 -0
  13. test/cache-ad0c3feb522b00d2.arrow +3 -0
  14. test/cache-bb2f31305a09ac2e_00000_of_00002.arrow +3 -0
  15. test/cache-bb2f31305a09ac2e_00001_of_00002.arrow +3 -0
  16. test/cache-e38c0144724a4def.arrow +3 -0
  17. test/data-00000-of-00001.arrow +2 -2
  18. test/state.json +1 -1
  19. train_results.json +3 -3
  20. trainer_state.json +7 -7
  21. training_args.bin +1 -1
README.md CHANGED
@@ -18,7 +18,7 @@ should probably proofread and complete it, then remove this comment. -->
18
 
19
  This model is a fine-tuned version of [meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) on the None dataset.
20
  It achieves the following results on the evaluation set:
21
- - Loss: 2.5661
22
 
23
  ## Model description
24
 
@@ -53,7 +53,7 @@ The following hyperparameters were used during training:
53
 
54
  | Training Loss | Epoch | Step | Validation Loss |
55
  |:-------------:|:-----:|:----:|:---------------:|
56
- | 1.4265 | 1.0 | 1 | 2.5661 |
57
 
58
 
59
  ### Framework versions
 
18
 
19
  This model is a fine-tuned version of [meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) on the None dataset.
20
  It achieves the following results on the evaluation set:
21
+ - Loss: 2.8863
22
 
23
  ## Model description
24
 
 
53
 
54
  | Training Loss | Epoch | Step | Validation Loss |
55
  |:-------------:|:-----:|:----:|:---------------:|
56
+ | 1.4265 | 1.0 | 1 | 2.8863 |
57
 
58
 
59
  ### Framework versions
adapter_config.json CHANGED
@@ -20,13 +20,13 @@
20
  "rank_pattern": {},
21
  "revision": null,
22
  "target_modules": [
 
23
  "q_proj",
24
- "up_proj",
25
  "down_proj",
26
- "o_proj",
27
- "v_proj",
28
  "gate_proj",
29
- "k_proj"
 
30
  ],
31
  "task_type": "CAUSAL_LM",
32
  "use_dora": false,
 
20
  "rank_pattern": {},
21
  "revision": null,
22
  "target_modules": [
23
+ "o_proj",
24
  "q_proj",
 
25
  "down_proj",
26
+ "up_proj",
 
27
  "gate_proj",
28
+ "k_proj",
29
+ "v_proj"
30
  ],
31
  "task_type": "CAUSAL_LM",
32
  "use_dora": false,
all_results.json CHANGED
@@ -1,8 +1,13 @@
1
  {
2
  "epoch": 1.0,
 
 
 
 
 
3
  "train_loss": 1.4264578819274902,
4
- "train_runtime": 1.98,
5
  "train_samples": 4,
6
- "train_samples_per_second": 2.02,
7
- "train_steps_per_second": 0.505
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "eval_loss": 2.566110134124756,
4
+ "eval_runtime": 0.2859,
5
+ "eval_samples": 4,
6
+ "eval_samples_per_second": 13.992,
7
+ "eval_steps_per_second": 3.498,
8
  "train_loss": 1.4264578819274902,
9
+ "train_runtime": 2.0986,
10
  "train_samples": 4,
11
+ "train_samples_per_second": 1.906,
12
+ "train_steps_per_second": 0.477
13
  }
config.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "meta-llama/Meta-Llama-3-8B-Instruct",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 128000,
9
+ "eos_token_id": 128009,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 4096,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 14336,
14
+ "max_position_embeddings": 8192,
15
+ "model_type": "llama",
16
+ "num_attention_heads": 32,
17
+ "num_hidden_layers": 32,
18
+ "num_key_value_heads": 8,
19
+ "pretraining_tp": 1,
20
+ "quantization_config": {
21
+ "bnb_4bit_compute_dtype": "bfloat16",
22
+ "bnb_4bit_quant_type": "nf4",
23
+ "bnb_4bit_use_double_quant": false,
24
+ "llm_int8_enable_fp32_cpu_offload": false,
25
+ "llm_int8_has_fp16_weight": false,
26
+ "llm_int8_skip_modules": null,
27
+ "llm_int8_threshold": 6.0,
28
+ "load_in_4bit": true,
29
+ "load_in_8bit": false,
30
+ "quant_method": "bitsandbytes"
31
+ },
32
+ "rms_norm_eps": 1e-05,
33
+ "rope_scaling": null,
34
+ "rope_theta": 500000.0,
35
+ "tie_word_embeddings": false,
36
+ "torch_dtype": "bfloat16",
37
+ "transformers_version": "4.36.2",
38
+ "use_cache": true,
39
+ "vocab_size": 128256
40
+ }
dataset_dict.json CHANGED
@@ -1 +1 @@
1
- {"splits": ["train", "test"]}
 
1
+ {"splits": ["train", "test", "evaluation"]}
eval_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.0,
3
+ "eval_loss": 2.566110134124756,
4
+ "eval_runtime": 0.2859,
5
+ "eval_samples": 4,
6
+ "eval_samples_per_second": 13.992,
7
+ "eval_steps_per_second": 3.498
8
+ }
evaluation/data-00000-of-00001.arrow ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d32bef1c42293fadaad0794f163ccb1bb2d8f83df2968eb1544bc51d7d62b0c3
3
+ size 4384
evaluation/dataset_info.json ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "citation": "",
3
+ "description": "",
4
+ "features": {
5
+ "prompt": {
6
+ "dtype": "string",
7
+ "_type": "Value"
8
+ },
9
+ "chosen": [
10
+ {
11
+ "content": {
12
+ "dtype": "string",
13
+ "_type": "Value"
14
+ },
15
+ "role": {
16
+ "dtype": "string",
17
+ "_type": "Value"
18
+ }
19
+ }
20
+ ],
21
+ "rejected": {
22
+ "dtype": "null",
23
+ "_type": "Value"
24
+ },
25
+ "messages": [
26
+ {
27
+ "content": {
28
+ "dtype": "string",
29
+ "_type": "Value"
30
+ },
31
+ "role": {
32
+ "dtype": "string",
33
+ "_type": "Value"
34
+ }
35
+ }
36
+ ],
37
+ "score_chosen": {
38
+ "dtype": "int64",
39
+ "_type": "Value"
40
+ },
41
+ "score_rejected": {
42
+ "dtype": "int64",
43
+ "_type": "Value"
44
+ }
45
+ },
46
+ "homepage": "",
47
+ "license": ""
48
+ }
evaluation/state.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_data_files": [
3
+ {
4
+ "filename": "data-00000-of-00001.arrow"
5
+ }
6
+ ],
7
+ "_fingerprint": "a5cce1ec601ad10f",
8
+ "_format_columns": null,
9
+ "_format_kwargs": {},
10
+ "_format_type": null,
11
+ "_output_all_columns": false,
12
+ "_split": null
13
+ }
runs/May23_17-07-55_scai5.cs.ucla.edu/events.out.tfevents.1716509488.scai5.cs.ucla.edu.1145330.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1c4b77d8f08c8cf39b6b9b55ad6c248dfc40a5ec24dbec61071fc6c161207190
3
+ size 354
runs/May23_17-23-15_scai5.cs.ucla.edu/events.out.tfevents.1716510206.scai5.cs.ucla.edu.1148301.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fad893d887989d4eeb325a1b2c96192149d1e4462e595adb6541a71b07a4c1f2
3
+ size 5840
test/cache-6d56a9de8848869b.arrow ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7c60ecb6771979ecb6c71bdc5e56814a16ae51b130fd3ce7813c1206c4bb7394
3
+ size 464
test/cache-ad0c3feb522b00d2.arrow ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:230d4d5106c83a8eb49619c59478849b6c8979d83e572f24a96ba8009af0246c
3
+ size 1968
test/cache-bb2f31305a09ac2e_00000_of_00002.arrow ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd1858af16739f83b6da3ee7f3dd61a896f9b9a2b28f8399f637b2f3c0ac7888
3
+ size 904
test/cache-bb2f31305a09ac2e_00001_of_00002.arrow ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a57f6a414c4469eff7963f5af8ef9000ba75ed6aef2c8745c3eece87d38dd1e9
3
+ size 880
test/cache-e38c0144724a4def.arrow ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:753210810e41cb4a973ffda88cb96dd1a36631a5bf2c2b66a90e0926a42827a4
3
+ size 464
test/data-00000-of-00001.arrow CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:89b447fbcadcbc4ad282e3a02100b92af74635598251b6f14a6a045acb5259be
3
- size 6368
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:44c159a71342e18b02d8ab9af9db7a2aa56d9ed0fccea084cc32b6d22814cf9a
3
+ size 3904
test/state.json CHANGED
@@ -4,7 +4,7 @@
4
  "filename": "data-00000-of-00001.arrow"
5
  }
6
  ],
7
- "_fingerprint": "d43f8e3963fd18ec",
8
  "_format_columns": null,
9
  "_format_kwargs": {},
10
  "_format_type": null,
 
4
  "filename": "data-00000-of-00001.arrow"
5
  }
6
  ],
7
+ "_fingerprint": "fe438c5dd0a3efa5",
8
  "_format_columns": null,
9
  "_format_kwargs": {},
10
  "_format_type": null,
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
  "train_loss": 1.4264578819274902,
4
- "train_runtime": 1.98,
5
  "train_samples": 4,
6
- "train_samples_per_second": 2.02,
7
- "train_steps_per_second": 0.505
8
  }
 
1
  {
2
  "epoch": 1.0,
3
  "train_loss": 1.4264578819274902,
4
+ "train_runtime": 2.0986,
5
  "train_samples": 4,
6
+ "train_samples_per_second": 1.906,
7
+ "train_steps_per_second": 0.477
8
  }
trainer_state.json CHANGED
@@ -16,10 +16,10 @@
16
  },
17
  {
18
  "epoch": 1.0,
19
- "eval_loss": 2.566110134124756,
20
- "eval_runtime": 0.2634,
21
- "eval_samples_per_second": 15.184,
22
- "eval_steps_per_second": 3.796,
23
  "step": 1
24
  },
25
  {
@@ -27,9 +27,9 @@
27
  "step": 1,
28
  "total_flos": 32602467598336.0,
29
  "train_loss": 1.4264578819274902,
30
- "train_runtime": 1.98,
31
- "train_samples_per_second": 2.02,
32
- "train_steps_per_second": 0.505
33
  }
34
  ],
35
  "logging_steps": 5,
 
16
  },
17
  {
18
  "epoch": 1.0,
19
+ "eval_loss": 2.88632869720459,
20
+ "eval_runtime": 0.263,
21
+ "eval_samples_per_second": 7.604,
22
+ "eval_steps_per_second": 3.802,
23
  "step": 1
24
  },
25
  {
 
27
  "step": 1,
28
  "total_flos": 32602467598336.0,
29
  "train_loss": 1.4264578819274902,
30
+ "train_runtime": 2.0986,
31
+ "train_samples_per_second": 1.906,
32
+ "train_steps_per_second": 0.477
33
  }
34
  ],
35
  "logging_steps": 5,
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:782dfa2d331e6c9615faebbd7bbbb707d63345cb137deef1e6c6b6e9b31171be
3
  size 4920
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7f34f7e356f32afb3f6b92b747a154e8890a05871c5f1c11868b6a2373cc17df
3
  size 4920