MaziyarPanahi commited on
Commit
f75ebf5
1 Parent(s): 0cf0284
adapter/adapter_config.json DELETED
@@ -1,38 +0,0 @@
1
- {
2
- "alpha_pattern": {},
3
- "auto_mapping": null,
4
- "base_model_name_or_path": "v2ray/Mixtral-8x22B-v0.1",
5
- "bias": "none",
6
- "fan_in_fan_out": null,
7
- "inference_mode": true,
8
- "init_lora_weights": true,
9
- "layer_replication": null,
10
- "layers_pattern": null,
11
- "layers_to_transform": null,
12
- "loftq_config": {},
13
- "lora_alpha": 8,
14
- "lora_dropout": 0.05,
15
- "megatron_config": null,
16
- "megatron_core": "megatron.core",
17
- "modules_to_save": [
18
- "embed_tokens",
19
- "lm_head"
20
- ],
21
- "peft_type": "LORA",
22
- "r": 16,
23
- "rank_pattern": {},
24
- "revision": null,
25
- "target_modules": [
26
- "q_proj",
27
- "w2",
28
- "w3",
29
- "o_proj",
30
- "v_proj",
31
- "k_proj",
32
- "gate",
33
- "w1"
34
- ],
35
- "task_type": "CAUSAL_LM",
36
- "use_dora": false,
37
- "use_rslora": false
38
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
adapter/adapter_model.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:e7c3e6eac571b2034dd287ea2c037b3187e9ea2a77b00a0166a08708fc006e87
3
- size 3409573984
 
 
 
 
adapter/added_tokens.json DELETED
@@ -1,4 +0,0 @@
1
- {
2
- "<|im_end|>": 32000,
3
- "<|im_start|>": 32001
4
- }
 
 
 
 
 
adapter/config.json DELETED
@@ -1,46 +0,0 @@
1
- {
2
- "_name_or_path": "v2ray/Mixtral-8x22B-v0.1",
3
- "architectures": [
4
- "MixtralForCausalLM"
5
- ],
6
- "attention_dropout": 0.0,
7
- "bos_token_id": 1,
8
- "eos_token_id": 32000,
9
- "hidden_act": "silu",
10
- "hidden_size": 6144,
11
- "initializer_range": 0.02,
12
- "intermediate_size": 16384,
13
- "max_position_embeddings": 65536,
14
- "model_type": "mixtral",
15
- "num_attention_heads": 48,
16
- "num_experts_per_tok": 2,
17
- "num_hidden_layers": 56,
18
- "num_key_value_heads": 8,
19
- "num_local_experts": 8,
20
- "output_router_logits": false,
21
- "quantization_config": {
22
- "_load_in_4bit": true,
23
- "_load_in_8bit": false,
24
- "bnb_4bit_compute_dtype": "bfloat16",
25
- "bnb_4bit_quant_storage": "bfloat16",
26
- "bnb_4bit_quant_type": "nf4",
27
- "bnb_4bit_use_double_quant": true,
28
- "llm_int8_enable_fp32_cpu_offload": false,
29
- "llm_int8_has_fp16_weight": false,
30
- "llm_int8_skip_modules": null,
31
- "llm_int8_threshold": 6.0,
32
- "load_in_4bit": true,
33
- "load_in_8bit": false,
34
- "quant_method": "bitsandbytes"
35
- },
36
- "rms_norm_eps": 1e-05,
37
- "rope_theta": 1000000,
38
- "router_aux_loss_coef": 0.001,
39
- "router_jitter_noise": 0.0,
40
- "sliding_window": null,
41
- "tie_word_embeddings": false,
42
- "torch_dtype": "bfloat16",
43
- "transformers_version": "4.40.0.dev0",
44
- "use_cache": false,
45
- "vocab_size": 32002
46
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
adapter/special_tokens_map.json DELETED
@@ -1,24 +0,0 @@
1
- {
2
- "bos_token": {
3
- "content": "<s>",
4
- "lstrip": false,
5
- "normalized": false,
6
- "rstrip": false,
7
- "single_word": false
8
- },
9
- "eos_token": {
10
- "content": "<|im_end|>",
11
- "lstrip": false,
12
- "normalized": false,
13
- "rstrip": false,
14
- "single_word": false
15
- },
16
- "pad_token": "</s>",
17
- "unk_token": {
18
- "content": "<unk>",
19
- "lstrip": false,
20
- "normalized": false,
21
- "rstrip": false,
22
- "single_word": false
23
- }
24
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
adapter/tokenizer.model DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
3
- size 493443
 
 
 
 
adapter/tokenizer_config.json DELETED
@@ -1,61 +0,0 @@
1
- {
2
- "add_bos_token": true,
3
- "add_eos_token": false,
4
- "add_prefix_space": true,
5
- "added_tokens_decoder": {
6
- "0": {
7
- "content": "<unk>",
8
- "lstrip": false,
9
- "normalized": false,
10
- "rstrip": false,
11
- "single_word": false,
12
- "special": true
13
- },
14
- "1": {
15
- "content": "<s>",
16
- "lstrip": false,
17
- "normalized": false,
18
- "rstrip": false,
19
- "single_word": false,
20
- "special": true
21
- },
22
- "2": {
23
- "content": "</s>",
24
- "lstrip": false,
25
- "normalized": false,
26
- "rstrip": false,
27
- "single_word": false,
28
- "special": true
29
- },
30
- "32000": {
31
- "content": "<|im_end|>",
32
- "lstrip": false,
33
- "normalized": false,
34
- "rstrip": false,
35
- "single_word": false,
36
- "special": true
37
- },
38
- "32001": {
39
- "content": "<|im_start|>",
40
- "lstrip": false,
41
- "normalized": false,
42
- "rstrip": false,
43
- "single_word": false,
44
- "special": false
45
- }
46
- },
47
- "additional_special_tokens": [],
48
- "bos_token": "<s>",
49
- "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
50
- "clean_up_tokenization_spaces": false,
51
- "eos_token": "<|im_end|>",
52
- "legacy": true,
53
- "model_max_length": 1000000000000000019884624838656,
54
- "pad_token": "</s>",
55
- "sp_model_kwargs": {},
56
- "spaces_between_special_tokens": false,
57
- "tokenizer_class": "LlamaTokenizer",
58
- "unk_token": "<unk>",
59
- "use_default_system_prompt": false,
60
- "use_fast": true
61
- }