Initial commit

.gitattributes

@@ -0,0 +1,6 @@
+tokenizer.model filter=lfs diff=lfs merge=lfs -text
+clip_vision_embedding.bin filter=lfs diff=lfs merge=lfs -text
+dream_embedding.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model.bin filter=lfs diff=lfs merge=lfs -text
+stable_diffusion_head.bin filter=lfs diff=lfs merge=lfs -text
+training_args.bin filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,10 @@
+{
+  "<dream>": 32007,
+  "<dream_end>": 32002,
+  "<dream_start>": 32003,
+  "<im_end>": 32005,
+  "<im_patch>": 32004,
+  "<im_start>": 32001,
+  "<image>": 32006,
+  "[PAD]": 32000
+}

clip_vision_embedding.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34de1cf8ae92bce7e698ebdc57d1d85dcc7ef33f908fe7d44caaa6e3a17bc7d7
+size 614902217

config.json

@@ -0,0 +1,88 @@
+{
+  "architectures": [
+    "DreamLLMForCausalMLM"
+  ],
+  "attention_bias": false,
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "log_attentions": false,
+  "log_hidden_states": false,
+  "loss_scale_schedule": "none",
+  "loss_weight_lm": 1.0,
+  "loss_weight_vm": 10.0,
+  "max_position_embeddings": 2048,
+  "max_sequence_length": 2048,
+  "model_type": "dreamllm",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": -1,
+  "plugins_init_kwargs": {
+    "clip_vision_embedding": {
+      "_target_": "omni.models.dreamllm.modeling_plugins.CLIPVisionEmbedding",
+      "clip_vision_model_name_or_path": "openai/clip-vit-large-patch14",
+      "embed_hidden_size": 4096,
+      "freeze_clip_vision_model": true,
+      "freeze_embedding_layers": true,
+      "freeze_projector": false,
+      "local_files_only": false,
+      "pretrained_model_name_or_path": "none",
+      "projector_depth": 1,
+      "projector_type": "linear",
+      "select_layer": -2,
+      "use_additional_post_layernorm": false
+    },
+    "dream_embedding": {
+      "_target_": "omni.models.dreamllm.modeling_plugins.DreamEmbedding",
+      "embed_hidden_size": 4096,
+      "freeze_dream_queries": false,
+      "num_dream_queries": 64,
+      "pretrained_model_name_or_path": "none"
+    },
+    "stable_diffusion_head": {
+      "_target_": "omni.models.dreamllm.modeling_plugins.StableDiffusionHead",
+      "diffusion_name_or_path": "stabilityai/stable-diffusion-2-1-base",
+      "embed_hidden_size": 4096,
+      "freeze_projector": false,
+      "freeze_unet": true,
+      "freeze_vae": true,
+      "local_files_only": false,
+      "pretrained_model_name_or_path": "none",
+      "projector_depth": 1,
+      "projector_type": "linear"
+    }
+  },
+  "plugins_type": {
+    "clip_vision_embedding": "embedding",
+    "dream_embedding": "embedding",
+    "stable_diffusion_head": "head"
+  },
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "special_tokens2ids_dict": {
+    "</s>": 2,
+    "<s>": 1,
+    "<unk>": 0,
+    "[PAD]": 32000,
+    "additional_special_tokens": {
+      "<dream>": 32007,
+      "<dream_end>": 32002,
+      "<dream_start>": 32003,
+      "<im_end>": 32005,
+      "<im_patch>": 32004,
+      "<im_start>": 32001,
+      "<image>": 32006
+    }
+  },
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.35.2",
+  "use_cache": true,
+  "vocab_size": 32008
+}

dream_embedding.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38bc346c0e16c9ed3dfef5f7656f631259db18c131a882e00ea3deb2ddfbfe7a
+size 525187

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "pad_token_id": 0,
+  "transformers_version": "4.35.2"
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2719d78f070e0808e0091362bba0d5c79015cc956a4722b7a6d353b1d6363d48
+size 13477271009

special_tokens_map.json

@@ -0,0 +1,39 @@
+{
+  "additional_special_tokens": [
+    "<im_start>",
+    "<dream_end>",
+    "<dream_start>",
+    "<im_patch>",
+    "<im_end>",
+    "<image>",
+    "<dream>"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

stable_diffusion_head.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:820456700dc1ab6656ac4e156977f8b85fa0004a0306b27208e79f095c38c744
+size 1907851377

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,116 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<im_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
+      "content": "<dream_end>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32003": {
+      "content": "<dream_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32004": {
+      "content": "<im_patch>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32005": {
+      "content": "<im_end>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32006": {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32007": {
+      "content": "<dream>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<im_start>",
+    "<dream_end>",
+    "<dream_start>",
+    "<im_patch>",
+    "<im_end>",
+    "<image>",
+    "<dream>"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "[PAD]",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false,
+  "use_fast": true
+}