Upload model

config.json

@@ -0,0 +1,32 @@
+{
+  "_name_or_path": "crumb/shrink-init",
+  "activation_function": "silu",
+  "architectures": [
+    "ShrinkModelForCausalLM"
+  ],
+  "auto_map": {
+    "AutoConfig": "crumb/shrink-init--configuration_shrink.ShrinkConfig",
+    "AutoModelForCausalLM": "crumb/shrink-init--modeling_shrink.ShrinkModelForCausalLM"
+  },
+  "bos_token_id": 1,
+  "combined_qkv": true,
+  "eos_token_id": 2,
+  "hidden_size": 1024,
+  "hidden_size_0": 8192,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_dropout_prob": 0.0,
+  "layer_norm_epsilon": 1e-06,
+  "lm_head_bias": false,
+  "max_position_embeddings": 2048,
+  "model_type": "shrink",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 14,
+  "projection_bias": true,
+  "qk_hidden_size": null,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.36.2",
+  "use_bias": false,
+  "use_cache": false,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.36.2"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0005fd1c9e9af0db685abb7630def7d4d5a2d6528e2f0683368a1498310c4793
+size 1027759938

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c75679b86904686700edbe306217176fa25214dacc0a6a4413a10d751df32b0
+size 524288128

model.safetensors.index.json

@@ -0,0 +1,144 @@
+{
+  "metadata": {
+    "total_size": 1552033794
+  },
+  "weight_map": {
+    "lm_head.0.bias": "model-00001-of-00002.safetensors",
+    "lm_head.0.weight": "model-00001-of-00002.safetensors",
+    "lm_head.1.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.0.attn.out.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.ffn.down_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.ffn.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.ffn.up_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.ln1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.0.ln1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.ln2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.0.ln2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.attn.out.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.ffn.down_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.ffn.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.ffn.up_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.ln1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.1.ln1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.ln2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.1.ln2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.attn.out.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.ffn.down_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.ffn.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.ffn.up_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.ln1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.10.ln1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.ln2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.10.ln2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.attn.out.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.ffn.down_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.ffn.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.ffn.up_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.ln1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.11.ln1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.ln2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.11.ln2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.attn.out.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.ffn.down_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.ffn.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.ffn.up_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.ln1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.12.ln1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.ln2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.12.ln2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.attn.out.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.ffn.down_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.ffn.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.ffn.up_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.ln1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.13.ln1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.ln2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.13.ln2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.attn.out.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.ffn.down_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.ffn.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.ffn.up_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.ln1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.2.ln1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.ln2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.2.ln2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.attn.out.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.ffn.down_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.ffn.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.ffn.up_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.ln1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.3.ln1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.ln2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.3.ln2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.attn.out.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.ffn.down_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.ffn.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.ffn.up_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.ln1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.4.ln1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.ln2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.4.ln2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.attn.out.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.ffn.down_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.ffn.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.ffn.up_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.ln1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.5.ln1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.ln2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.5.ln2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.attn.out.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.ffn.down_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.ffn.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.ffn.up_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.ln1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.6.ln1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.ln2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.6.ln2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.attn.out.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.ffn.down_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.ffn.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.ffn.up_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.ln1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.7.ln1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.ln2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.7.ln2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.attn.out.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.ffn.down_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.ffn.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.ffn.up_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.ln1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.8.ln1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.ln2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.8.ln2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.attn.out.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.ffn.down_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.ffn.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.ffn.up_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.ln1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.9.ln1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.ln2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.9.ln2.weight": "model-00001-of-00002.safetensors",
+    "transformer.ln_f.bias": "model-00001-of-00002.safetensors",
+    "transformer.ln_f.weight": "model-00001-of-00002.safetensors",
+    "transformer.wln.bias": "model-00001-of-00002.safetensors",
+    "transformer.wln.weight": "model-00001-of-00002.safetensors",
+    "transformer.wpe.scale_factor": "model-00001-of-00002.safetensors",
+    "transformer.wte.0.weight": "model-00001-of-00002.safetensors",
+    "transformer.wte.1.bias": "model-00001-of-00002.safetensors",
+    "transformer.wte.1.weight": "model-00001-of-00002.safetensors"
+  }
+}