3v324v23 commited on
Commit
72e9344
1 Parent(s): 9cbece2

model galore

Browse files
README.md CHANGED
@@ -1,3 +1,141 @@
1
  ---
2
  license: apache-2.0
 
 
 
 
 
 
3
  ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
  license: apache-2.0
3
+ base_model: HuggingFaceTB/cosmo-1b
4
+ tags:
5
+ - generated_from_trainer
6
+ model-index:
7
+ - name: galore-out
8
+ results: []
9
  ---
10
+
11
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
12
+ should probably proofread and complete it, then remove this comment. -->
13
+
14
+ [<img src="https://raw.githubusercontent.com/OpenAccess-AI-Collective/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/OpenAccess-AI-Collective/axolotl)
15
+ <details><summary>See axolotl config</summary>
16
+
17
+ axolotl version: `0.4.0`
18
+ ```yaml
19
+ base_model: HuggingFaceTB/cosmo-1b
20
+ model_type: LlamaForCausalLM
21
+ tokenizer_type: LlamaTokenizer
22
+
23
+ load_in_8bit: false
24
+ load_in_4bit: false
25
+ strict: false
26
+
27
+ datasets:
28
+ - path: Vezora/Tested-22k-Python-Alpaca
29
+ type: alpaca
30
+ dataset_prepared_path: prepared-galore
31
+ val_set_size: 0.05
32
+ output_dir: ./galore-out
33
+
34
+ sequence_len: 2048
35
+ sample_packing: true
36
+ pad_to_sequence_len: true
37
+
38
+ adapter:
39
+ lora_model_dir:
40
+ lora_r:
41
+ lora_alpha:
42
+ lora_dropout:
43
+ lora_target_linear:
44
+ lora_fan_in_fan_out:
45
+
46
+ wandb_project: cosmo-python-galore
47
+ wandb_entity:
48
+ wandb_watch:
49
+ wandb_name:
50
+ wandb_log_model:
51
+
52
+ gradient_accumulation_steps: 4
53
+ micro_batch_size: 2
54
+ num_epochs: 1
55
+ optimizer: galore_adamw_8bit
56
+ lr_scheduler: cosine
57
+ learning_rate: 0.0005
58
+
59
+ optim_target_modules:
60
+ - self_attn # for llama
61
+ - mlp
62
+
63
+
64
+ train_on_inputs: false
65
+ group_by_length: false
66
+ bf16: auto
67
+ fp16:
68
+ tf32: false
69
+
70
+ gradient_checkpointing: true
71
+ early_stopping_patience:
72
+ resume_from_checkpoint:
73
+ local_rank:
74
+ logging_steps: 1
75
+ xformers_attention:
76
+ flash_attention: true
77
+
78
+ warmup_steps: 10
79
+ evals_per_epoch: 4
80
+ saves_per_epoch: 1
81
+ debug:
82
+ deepspeed:
83
+ weight_decay: 0.0
84
+ fsdp:
85
+ fsdp_config:
86
+ special_tokens:
87
+
88
+ ```
89
+
90
+ </details><br>
91
+
92
+ # galore-out
93
+
94
+ This model is a fine-tuned version of [HuggingFaceTB/cosmo-1b](https://huggingface.co/HuggingFaceTB/cosmo-1b) on the None dataset.
95
+ It achieves the following results on the evaluation set:
96
+ - Loss: 0.3833
97
+
98
+ ## Model description
99
+
100
+ More information needed
101
+
102
+ ## Intended uses & limitations
103
+
104
+ More information needed
105
+
106
+ ## Training and evaluation data
107
+
108
+ More information needed
109
+
110
+ ## Training procedure
111
+
112
+ ### Training hyperparameters
113
+
114
+ The following hyperparameters were used during training:
115
+ - learning_rate: 0.0005
116
+ - train_batch_size: 2
117
+ - eval_batch_size: 2
118
+ - seed: 42
119
+ - gradient_accumulation_steps: 4
120
+ - total_train_batch_size: 8
121
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
122
+ - lr_scheduler_type: cosine
123
+ - lr_scheduler_warmup_steps: 10
124
+ - num_epochs: 1
125
+
126
+ ### Training results
127
+
128
+ | Training Loss | Epoch | Step | Validation Loss |
129
+ |:-------------:|:-----:|:----:|:---------------:|
130
+ | 0.6299 | 0.0 | 1 | 0.6469 |
131
+ | 0.4194 | 0.25 | 217 | 0.4653 |
132
+ | 0.3577 | 0.5 | 434 | 0.4068 |
133
+ | 0.4144 | 0.75 | 651 | 0.3833 |
134
+
135
+
136
+ ### Framework versions
137
+
138
+ - Transformers 4.40.0.dev0
139
+ - Pytorch 2.1.2+cu118
140
+ - Datasets 2.18.0
141
+ - Tokenizers 0.15.0
config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "HuggingFaceTB/cosmo-1b",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "eos_token_id": 2,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 2048,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 8192,
14
+ "max_position_embeddings": 2048,
15
+ "model_type": "llama",
16
+ "num_attention_heads": 16,
17
+ "num_hidden_layers": 24,
18
+ "num_key_value_heads": 16,
19
+ "pretraining_tp": 1,
20
+ "rms_norm_eps": 1e-05,
21
+ "rope_scaling": null,
22
+ "rope_theta": 10000.0,
23
+ "tie_word_embeddings": false,
24
+ "torch_dtype": "bfloat16",
25
+ "transformers_version": "4.40.0.dev0",
26
+ "use_cache": false,
27
+ "vocab_size": 32000
28
+ }
generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "do_sample": true,
5
+ "eos_token_id": 2,
6
+ "transformers_version": "4.40.0.dev0"
7
+ }
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:97d8deb89efe133140a772e95835a6051ca0ebc007f5b468fff984b0e9caa2ad
3
+ size 3483644214
special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "</s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
3
+ size 493443
tokenizer_config.json ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "add_prefix_space": true,
5
+ "added_tokens_decoder": {
6
+ "0": {
7
+ "content": "<unk>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false,
12
+ "special": true
13
+ },
14
+ "1": {
15
+ "content": "<s>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false,
20
+ "special": true
21
+ },
22
+ "2": {
23
+ "content": "</s>",
24
+ "lstrip": false,
25
+ "normalized": false,
26
+ "rstrip": false,
27
+ "single_word": false,
28
+ "special": true
29
+ }
30
+ },
31
+ "additional_special_tokens": [],
32
+ "bos_token": "<s>",
33
+ "clean_up_tokenization_spaces": false,
34
+ "eos_token": "</s>",
35
+ "legacy": true,
36
+ "model_max_length": 1000000000000000019884624838656,
37
+ "pad_token": "</s>",
38
+ "sp_model_kwargs": {},
39
+ "spaces_between_special_tokens": false,
40
+ "tokenizer_class": "LlamaTokenizer",
41
+ "unk_token": "<unk>",
42
+ "use_default_system_prompt": false,
43
+ "use_fast": true
44
+ }