Upload 39 files

config.json

@@ -0,0 +1,35 @@
+{
+  "_name_or_path": "ahxt/LiteLlama-460M-1T",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layernorm_epsilon": 1e-05,
+  "max_position_embeddings": 1024,
+  "model_type": "mixtral",
+  "num_attention_heads": 16,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "num_local_experts": 512,
+  "output_router_logits": false,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "router_aux_loss_coef": 0.001,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.37.0.dev0",
+  "use_cache": true,
+  "vocab_size": 50304
+}

model-00001-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc4c30f0b734a31a16e6616ebf369cbbeec64c833ee70932105a34f772b42544
+size 9995758448

model-00002-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac9d2b0c074a47649cde03c0bb0617f27755dc7527682d1d38edff2a46914292
+size 9999377632

model-00003-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:621f19863c607f829228a050dffde7a5fb76d3fba923c33301d4463bc0ff7dab
+size 9999377632

model-00004-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d23f36428900f02900dc2531ea6a1806c6237e5b42bcbd94383eb7ebb831b7e
+size 9999377520

model-00005-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0f2781bf1ca5cc69d9db77875372710b79954340771b34a4ea87d5fe80123b3
+size 9999378448

model-00006-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e767cc5f6c7931acad370c67bc9615a04c15daaf4be8f3c110e2736ff50bec39
+size 9999378824

model-00007-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b08485f9c33ca372d07d3a75a905b2d60ed2aa2b7954ebfedf74e7d38802f96b
+size 9999378712

model-00008-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d243111db7f644fd38d65d3ef4db486f418d9be7447971c36cb1d18f5946fd2a
+size 9999378824

model-00009-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f80b51ce01743bdf48fe3eb2b172eeadcf47964e9256b193e95affae4a689496
+size 9999378824

model-00010-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9441fc462dfc0b982ef2a8ad15b2945dfa5ef27c63c03444344217463880116
+size 9999378712

model-00011-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d74e0b6fb4b39fac8673260f8eada1deae399e1d6c382c1b6be2498a9495d504
+size 9999378024

model-00012-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11c4153f4d3975eb66c031e610736dd98f56a3a0c40ee84c9cfe6a8580319cb9
+size 9999377632

model-00013-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d85f2a65a3d3a483d7794217b9ca3d792c7e284aa4a642b0edb9cf627ee897e
+size 9999377520

model-00014-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02eab7ad832eaa569604240fa713dc7105d4831a2068361b0348362cde858dc2
+size 9999377632

model-00015-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:733210ab9a820acf5053d94968feb6802a8fd5ea3be334e893472d88cf704c24
+size 9999378080

model-00016-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:553bec73fb656c07a51141d5eeffe7aeb7a9b04e85b6f6c1cb8b511b51a1495a
+size 9999378712

model-00017-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c92de3c49c5a766c55c76854f5001a000542f706c97dc67dffa516329c185f25
+size 9999378824

model-00018-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c0abcd986520a648aad7c025e9cb3ff495bea461b74207ad80424b4162ee2d2
+size 9999378824

model-00019-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc94d9c543391f9b978df723c220124b1a3252ed72def81c2bbaad26339b302d
+size 9999378720

model-00020-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f06a4f03ef54dbaf51503fa48364f02c2842220c60dab5483e3da3d8d1753e1d
+size 9999378816

model-00021-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fe40221f49d4af66c1f226f3e86946420c69f2b420b5ac28733d4bb2062268f
+size 9999378392

model-00022-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a85ced77657b19c1ce76d29b807049f7491176ebfea7b06fa07ea90ecbf1b5ae
+size 9999377536

model-00023-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a768c4fdaa149b06647017f5736034f4baaedd269df72572f75825afd7eca6f
+size 9999377616

model-00024-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af738a2ee159b59b03e0b87e4e2034a7e176cdf4780c4eac7e695484602fa8bf
+size 9999377632

model-00025-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:132304a4571f098282b4916e3b8a13a583ca7c0a5328ad77be6cb273be19a2d7
+size 9999377624

model-00026-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c907e90fda69f7bf3d44a30854329d4cfde2bff26ab2982e0ee75310fb1207d
+size 9999378800

model-00027-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9d32ae78add0604040a0a605d98c40abbb2aaa8797e1b4a9df4dfcb529fe0ae
+size 9999378824

model-00028-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a74d281b33598b9cd88b4b57ab81ba6e854edd7c3216ac28a82e9e9686bdae9
+size 9999378744

model-00029-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59de471593fa46ab3ec41344988ea3a4f04ffa1c75066222a7db0e3d6e201f4b
+size 9999378792

model-00030-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd7985d27d636866980dfb6dcdde51dd5d93f65783834862db92e915364f085
+size 9999378824

model-00031-of-00031.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05870ee02ab60261e71c8864dcc8fc65497ba11994a831d7ed8219453fdb50c1
+size 9609366048

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "#",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,38 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "1": {
+      "content": "\"",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "#",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "#",
+  "errors": "replace",
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}