winglian commited on
Commit
5e13925
1 Parent(s): 4a0083d

checkpoint-1053

Browse files
config.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "/home/sgugger/tmp/llama/llama-13b/",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "bos_token_id": 1,
7
+ "eos_token_id": 2,
8
+ "hidden_act": "silu",
9
+ "hidden_size": 5120,
10
+ "initializer_range": 0.02,
11
+ "intermediate_size": 13824,
12
+ "max_position_embeddings": 2048,
13
+ "max_sequence_length": 2048,
14
+ "model_type": "llama",
15
+ "num_attention_heads": 40,
16
+ "num_hidden_layers": 40,
17
+ "pad_token_id": 0,
18
+ "rms_norm_eps": 1e-06,
19
+ "tie_word_embeddings": false,
20
+ "torch_dtype": "float16",
21
+ "transformers_version": "4.28.0.dev0",
22
+ "use_cache": true,
23
+ "vocab_size": 32000
24
+ }
configs/minotaur.yml ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ base_model: huggyllama/llama-13b
2
+ base_model_config: huggyllama/llama-13b
3
+ model_type: LlamaForCausalLM
4
+ tokenizer_type: LlamaTokenizer
5
+ load_in_8bit: false
6
+ load_in_4bit: false
7
+ gptq: false
8
+ strict: false
9
+ push_dataset_to_hub: winglian
10
+ hf_use_auth_token: true
11
+ datasets:
12
+ - path: winglian/evals
13
+ data_files:
14
+ - hf/ARC-Challenge.jsonl
15
+ - hf/ARC-Easy.jsonl
16
+ - hf/riddle_sense.jsonl
17
+ type: explainchoice:chat
18
+ - path: winglian/evals
19
+ data_files:
20
+ - hf/gsm8k.jsonl
21
+ - hf/winogrande.jsonl
22
+ type: alpaca_chat.load_qa
23
+ - path: winglian/evals
24
+ data_files:
25
+ - custom/n_task.jsonl
26
+ - custom/misconceptions.jsonl
27
+ - custom/context_insensitivity.jsonl
28
+ type: alpaca_chat
29
+ - path: camel-ai/math
30
+ type: alpaca_chat.load_camel_ai
31
+ - path: camel-ai/biology
32
+ type: alpaca_chat.load_camel_ai
33
+ - path: camel-ai/physics
34
+ type: alpaca_chat.load_camel_ai
35
+ - path: camel-ai/chemistry
36
+ type: alpaca_chat.load_camel_ai
37
+ - path: winglian/evals
38
+ data_files:
39
+ - custom/in_context_qa.jsonl
40
+ type: context_qa
41
+ - path: winglian/evals
42
+ data_files:
43
+ - custom/in_context_qa.jsonl
44
+ type: context_qa.load_404
45
+ - path: winglian/evals
46
+ data_files:
47
+ - custom/jokes_explained_500up.jsonl
48
+ type: sharegpt_jokes
49
+ - path: winglian/evals
50
+ data_files:
51
+ - custom/classify-self-chat.sharegpt.jsonl
52
+ - custom/coding-self-chat.sharegpt.jsonl
53
+ - custom/prose-gpt4.sharegpt.jsonl
54
+ - custom/prose-rewrite-gpt4.sharegpt.jsonl
55
+ type: sharegpt_simple
56
+ - path: winglian/evals
57
+ data_files:
58
+ - openai/tldr.jsonl
59
+ type: summarizetldr:chat
60
+ - path: winglian/evals
61
+ data_files:
62
+ - hellaswag/hellaswag.jsonl
63
+ type: explainchoice:chat
64
+ shards: 60
65
+ - path: metaeval/ScienceQA_text_only
66
+ type: concisechoice:chat
67
+ shards: 13
68
+ - path: teknium/GPTeacher-General-Instruct
69
+ data_files: gpt4-instruct-similarity-0.6-dataset.json
70
+ type: gpteacher:chat
71
+ - path: QingyiSi/Alpaca-CoT
72
+ data_files:
73
+ - Chain-of-Thought/formatted_cot_data/aqua_train.json
74
+ - Chain-of-Thought/formatted_cot_data/creak_train.json
75
+ - Chain-of-Thought/formatted_cot_data/ecqa_train.json
76
+ - Chain-of-Thought/formatted_cot_data/esnli_train.json
77
+ - Chain-of-Thought/formatted_cot_data/gsm8k_train.json
78
+ - Chain-of-Thought/formatted_cot_data/qasc_train.json
79
+ - Chain-of-Thought/formatted_cot_data/qed_train.json
80
+ - Chain-of-Thought/formatted_cot_data/sensemaking_train.json
81
+ - Chain-of-Thought/formatted_cot_data/strategyqa_train.json
82
+ - GPTeacher/Roleplay/formatted_roleplay-similarity_0.6-instruct-dataset.json
83
+ type: alpaca_chat
84
+ - path: ehartford/WizardLM_alpaca_evol_instruct_70k_unfiltered
85
+ type: alpaca:chat
86
+ dataset_prepared_path: last_run_prepared
87
+ val_set_size: 0.01
88
+ adapter:
89
+ lora_model_dir:
90
+ sequence_len: 2048
91
+ max_packed_sequence_len: 2048
92
+ lora_r:
93
+ lora_alpha:
94
+ lora_dropout:
95
+ lora_target_modules:
96
+ lora_target_linear: true
97
+ lora_fan_in_fan_out:
98
+ wandb_project: minotaur-13b
99
+ wandb_watch:
100
+ wandb_run_id:
101
+ wandb_log_model:
102
+ output_dir: ./minotaur-13b
103
+ gradient_accumulation_steps: 1
104
+ micro_batch_size: 12
105
+ num_epochs: 12
106
+ optimizer: adamw_bnb_8bit
107
+ torchdistx_path:
108
+ lr_scheduler: cosine
109
+ learning_rate: 0.00001
110
+ train_on_inputs: false
111
+ group_by_length: false
112
+ bf16: true
113
+ fp16: false
114
+ tf32: true
115
+ gradient_checkpointing: true
116
+ early_stopping_patience:
117
+ resume_from_checkpoint:
118
+ local_rank:
119
+ logging_steps: 1
120
+ xformers_attention: true
121
+ flash_attention:
122
+ gptq_groupsize:
123
+ gptq_model_v1:
124
+ warmup_steps: 100
125
+ eval_steps: 27
126
+ save_steps: 27
127
+ load_best_model_at_end: false
128
+ save_total_limit: 10
129
+ debug:
130
+ deepspeed:
131
+ weight_decay: 0.00001
132
+ fsdp:
133
+ - full_shard
134
+ - auto_wrap
135
+ fsdp_config:
136
+ fsdp_offload_params: true
137
+ fsdp_transformer_layer_cls_to_wrap: LlamaDecoderLayer
138
+ special_tokens:
139
+ bos_token: "<s>"
140
+ eos_token: "</s>"
141
+ unk_token: "<unk>"
142
+
generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "pad_token_id": 0,
6
+ "transformers_version": "4.28.0.dev0"
7
+ }
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d78e29042e373d7ed73276dfc02a98edcb8e2fcb6f04507bdc9d469f09df2a37
3
+ size 26031937261
special_tokens_map.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": true,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": true,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "unk_token": {
17
+ "content": "<unk>",
18
+ "lstrip": false,
19
+ "normalized": true,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ }
23
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
+ size 499723
tokenizer_config.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "bos_token": {
5
+ "__type": "AddedToken",
6
+ "content": "<s>",
7
+ "lstrip": false,
8
+ "normalized": true,
9
+ "rstrip": false,
10
+ "single_word": false
11
+ },
12
+ "clean_up_tokenization_spaces": false,
13
+ "eos_token": {
14
+ "__type": "AddedToken",
15
+ "content": "</s>",
16
+ "lstrip": false,
17
+ "normalized": true,
18
+ "rstrip": false,
19
+ "single_word": false
20
+ },
21
+ "model_max_length": 2048,
22
+ "pad_token": null,
23
+ "sp_model_kwargs": {},
24
+ "tokenizer_class": "LlamaTokenizer",
25
+ "unk_token": {
26
+ "__type": "AddedToken",
27
+ "content": "<unk>",
28
+ "lstrip": false,
29
+ "normalized": true,
30
+ "rstrip": false,
31
+ "single_word": false
32
+ }
33
+ }