alexhafnerd commited on
Commit
79149bb
0 Parent(s):

Initial commit

Browse files
.gitattributes ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ clip_vision_embedding.bin filter=lfs diff=lfs merge=lfs -text
2
+ dream_embedding.bin filter=lfs diff=lfs merge=lfs -text
3
+ pytorch_model.bin filter=lfs diff=lfs merge=lfs -text
4
+ stable_diffusion_head.bin filter=lfs diff=lfs merge=lfs -text
5
+ tokenizer.model filter=lfs diff=lfs merge=lfs -text
6
+ pytorch_model_back.bin filter=lfs diff=lfs merge=lfs -text
added_tokens.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "<dream>": 32007,
3
+ "<dream_end>": 32002,
4
+ "<dream_start>": 32003,
5
+ "<im_end>": 32005,
6
+ "<im_patch>": 32004,
7
+ "<im_start>": 32001,
8
+ "<image>": 32006,
9
+ "[PAD]": 32000
10
+ }
clip_vision_embedding.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9278c079148131df640e45f35d3dc9c26d09443c2eb5b45b4becedb0042974bf
3
+ size 614896585
config.json ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "DreamLLMForCausalMLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "bos_token_id": 0,
7
+ "eos_token_id": 1,
8
+ "hidden_act": "silu",
9
+ "hidden_size": 4096,
10
+ "initializer_range": 0.02,
11
+ "intermediate_size": 11008,
12
+ "log_attentions": false,
13
+ "log_hidden_states": false,
14
+ "loss_scale_schedule": "none",
15
+ "loss_weight_lm": 1.0,
16
+ "loss_weight_vm": 10.0,
17
+ "max_position_embeddings": 2048,
18
+ "max_sequence_length": 2048,
19
+ "model_type": "dreamllm",
20
+ "num_attention_heads": 32,
21
+ "num_hidden_layers": 32,
22
+ "num_key_value_heads": 32,
23
+ "pad_token_id": -1,
24
+ "plugins_init_kwargs": {
25
+ "clip_vision_embedding": {
26
+ "_target_": "omni.models.dreamllm.modeling_plugins.CLIPVisionEmbedding",
27
+ "clip_vision_model_name_or_path": "openai/clip-vit-large-patch14",
28
+ "embed_hidden_size": 4096,
29
+ "freeze_clip_vision_model": true,
30
+ "freeze_embedding_layers": true,
31
+ "freeze_projector": false,
32
+ "local_files_only": true,
33
+ "pretrained_model_name_or_path": "none",
34
+ "projector_depth": 1,
35
+ "projector_type": "linear",
36
+ "select_layer": -2,
37
+ "use_additional_post_layernorm": false
38
+ },
39
+ "dream_embedding": {
40
+ "_target_": "omni.models.dreamllm.modeling_plugins.DreamEmbedding",
41
+ "embed_hidden_size": 4096,
42
+ "freeze_dream_queries": false,
43
+ "num_dream_queries": 64,
44
+ "pretrained_model_name_or_path": "none"
45
+ },
46
+ "stable_diffusion_head": {
47
+ "_target_": "omni.models.dreamllm.modeling_plugins.StableDiffusionHead",
48
+ "diffusion_name_or_path": "stabilityai/stable-diffusion-2-1-base",
49
+ "embed_hidden_size": 4096,
50
+ "freeze_projector": false,
51
+ "freeze_unet": true,
52
+ "freeze_vae": true,
53
+ "local_files_only": true,
54
+ "pretrained_model_name_or_path": "none",
55
+ "projector_depth": 1,
56
+ "projector_type": "linear"
57
+ }
58
+ },
59
+ "plugins_type": {
60
+ "clip_vision_embedding": "embedding",
61
+ "dream_embedding": "embedding",
62
+ "stable_diffusion_head": "head"
63
+ },
64
+ "pretraining_tp": 1,
65
+ "rms_norm_eps": 1e-06,
66
+ "rope_scaling": null,
67
+ "rope_theta": 10000.0,
68
+ "special_tokens2ids_dict": {
69
+ "</s>": 2,
70
+ "<s>": 1,
71
+ "<unk>": 0,
72
+ "[PAD]": 32000,
73
+ "additional_special_tokens": {
74
+ "<dream>": 32007,
75
+ "<dream_end>": 32002,
76
+ "<dream_start>": 32003,
77
+ "<im_end>": 32005,
78
+ "<im_patch>": 32004,
79
+ "<im_start>": 32001,
80
+ "<image>": 32006
81
+ }
82
+ },
83
+ "tie_word_embeddings": false,
84
+ "torch_dtype": "bfloat16",
85
+ "transformers_version": "4.35.2",
86
+ "use_cache": true,
87
+ "vocab_size": 32008
88
+ }
dream_embedding.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:33f3d3a9253888ac871ad48114c0605c72916db997fec6cb3a5df0f2b29d2fe7
3
+ size 525123
generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 0,
4
+ "eos_token_id": 1,
5
+ "pad_token_id": 0,
6
+ "transformers_version": "4.35.2"
7
+ }
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d03418a0b76c463790c1406381c1ab309a8d0392a20b58bc5d33435b789fc9b6
3
+ size 13477250085
special_tokens_map.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<im_start>",
4
+ "<dream_end>",
5
+ "<dream_start>",
6
+ "<im_patch>",
7
+ "<im_end>",
8
+ "<image>",
9
+ "<dream>"
10
+ ],
11
+ "bos_token": {
12
+ "content": "<s>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false
17
+ },
18
+ "eos_token": {
19
+ "content": "</s>",
20
+ "lstrip": false,
21
+ "normalized": false,
22
+ "rstrip": false,
23
+ "single_word": false
24
+ },
25
+ "pad_token": {
26
+ "content": "[PAD]",
27
+ "lstrip": false,
28
+ "normalized": false,
29
+ "rstrip": false,
30
+ "single_word": false
31
+ },
32
+ "unk_token": {
33
+ "content": "<unk>",
34
+ "lstrip": false,
35
+ "normalized": false,
36
+ "rstrip": false,
37
+ "single_word": false
38
+ }
39
+ }
stable_diffusion_head.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e35ffb5440055ab8ed76e4dcf2afe3b29e78d9d7d1a2e79f1a855ce776eba822
3
+ size 1907838193
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
+ size 499723
tokenizer_config.json ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<unk>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "1": {
14
+ "content": "<s>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "2": {
22
+ "content": "</s>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "32000": {
30
+ "content": "[PAD]",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ },
37
+ "32001": {
38
+ "content": "<im_start>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": true
44
+ },
45
+ "32002": {
46
+ "content": "<dream_end>",
47
+ "lstrip": false,
48
+ "normalized": false,
49
+ "rstrip": false,
50
+ "single_word": false,
51
+ "special": true
52
+ },
53
+ "32003": {
54
+ "content": "<dream_start>",
55
+ "lstrip": false,
56
+ "normalized": false,
57
+ "rstrip": false,
58
+ "single_word": false,
59
+ "special": true
60
+ },
61
+ "32004": {
62
+ "content": "<im_patch>",
63
+ "lstrip": false,
64
+ "normalized": false,
65
+ "rstrip": false,
66
+ "single_word": false,
67
+ "special": true
68
+ },
69
+ "32005": {
70
+ "content": "<im_end>",
71
+ "lstrip": false,
72
+ "normalized": false,
73
+ "rstrip": false,
74
+ "single_word": false,
75
+ "special": true
76
+ },
77
+ "32006": {
78
+ "content": "<image>",
79
+ "lstrip": false,
80
+ "normalized": false,
81
+ "rstrip": false,
82
+ "single_word": false,
83
+ "special": true
84
+ },
85
+ "32007": {
86
+ "content": "<dream>",
87
+ "lstrip": false,
88
+ "normalized": false,
89
+ "rstrip": false,
90
+ "single_word": false,
91
+ "special": true
92
+ }
93
+ },
94
+ "additional_special_tokens": [
95
+ "<im_start>",
96
+ "<dream_end>",
97
+ "<dream_start>",
98
+ "<im_patch>",
99
+ "<im_end>",
100
+ "<image>",
101
+ "<dream>"
102
+ ],
103
+ "bos_token": "<s>",
104
+ "clean_up_tokenization_spaces": false,
105
+ "eos_token": "</s>",
106
+ "legacy": true,
107
+ "model_max_length": 2048,
108
+ "pad_token": "[PAD]",
109
+ "padding_side": "right",
110
+ "sp_model_kwargs": {},
111
+ "spaces_between_special_tokens": false,
112
+ "tokenizer_class": "LlamaTokenizer",
113
+ "unk_token": "<unk>",
114
+ "use_default_system_prompt": false,
115
+ "use_fast": true
116
+ }