Upload model

config.json

@@ -3,7 +3,7 @@
   "alibi": false,
   "apply_residual_connection_post_layernorm": false,
   "architectures": [
-    "RWForCausalLM"
+    "RWModel"
   ],
   "attention_dropout": 0.0,
   "auto_map": {
@@ -26,17 +26,6 @@
   "n_head": 71,
   "n_layer": 32,
   "parallel_attn": true,
-  "quantization_config": {
-    "bnb_4bit_compute_dtype": "float16",
-    "bnb_4bit_quant_type": "nf4",
-    "bnb_4bit_use_double_quant": true,
-    "llm_int8_enable_fp32_cpu_offload": false,
-    "llm_int8_has_fp16_weight": false,
-    "llm_int8_skip_modules": null,
-    "llm_int8_threshold": 6.0,
-    "load_in_4bit": false,
-    "load_in_8bit": true
-  },
   "torch_dtype": "float16",
   "transformers_version": "4.31.0.dev0",
   "use_cache": true,

pytorch_model-00001-of-00002.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfff9b8566052f545c7dfa4b205d232e379895c302b2a4133857b665dc0293f9
+size 9951026529

pytorch_model-00002-of-00002.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bff80eaf11240587ffe4ca44b23fff07bb393f7ee2a3c8b7a150da365d5f044a
+size 3892482449

pytorch_model.bin.index.json

@@ -0,0 +1,202 @@
+{
+  "metadata": {
+    "total_size": 13843441408
+  },
+  "weight_map": {
+    "h.0.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.0.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.0.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.0.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.0.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.0.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.1.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.1.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.1.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.1.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.1.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.1.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.10.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.10.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.10.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.10.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.10.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.10.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.11.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.11.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.11.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.11.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.11.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.11.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.12.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.12.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.12.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.12.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.12.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.12.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.13.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.13.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.13.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.13.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.13.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.13.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.14.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.14.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.14.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.14.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.14.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.14.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.15.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.15.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.15.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.15.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.15.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.15.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.16.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.16.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.16.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.16.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.16.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.16.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.17.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.17.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.17.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.17.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.17.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.17.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.18.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.18.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.18.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.18.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.18.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.18.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.19.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.19.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.19.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.19.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.19.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.19.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.2.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.2.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.2.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.2.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.2.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.2.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.20.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.20.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.20.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.20.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.20.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.20.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.21.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.21.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.21.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.21.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.21.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.21.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.22.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.22.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.22.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin",
+    "h.22.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.22.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.22.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.23.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "h.23.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "h.23.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin",
+    "h.23.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin",
+    "h.23.self_attention.dense.weight": "pytorch_model-00002-of-00002.bin",
+    "h.23.self_attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin",
+    "h.24.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "h.24.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "h.24.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin",
+    "h.24.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin",
+    "h.24.self_attention.dense.weight": "pytorch_model-00002-of-00002.bin",
+    "h.24.self_attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin",
+    "h.25.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "h.25.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "h.25.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin",
+    "h.25.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin",
+    "h.25.self_attention.dense.weight": "pytorch_model-00002-of-00002.bin",
+    "h.25.self_attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin",
+    "h.26.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "h.26.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "h.26.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin",
+    "h.26.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin",
+    "h.26.self_attention.dense.weight": "pytorch_model-00002-of-00002.bin",
+    "h.26.self_attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin",
+    "h.27.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "h.27.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "h.27.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin",
+    "h.27.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin",
+    "h.27.self_attention.dense.weight": "pytorch_model-00002-of-00002.bin",
+    "h.27.self_attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin",
+    "h.28.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "h.28.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "h.28.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin",
+    "h.28.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin",
+    "h.28.self_attention.dense.weight": "pytorch_model-00002-of-00002.bin",
+    "h.28.self_attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin",
+    "h.29.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "h.29.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "h.29.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin",
+    "h.29.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin",
+    "h.29.self_attention.dense.weight": "pytorch_model-00002-of-00002.bin",
+    "h.29.self_attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin",
+    "h.3.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.3.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.3.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.3.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.3.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.3.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.30.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "h.30.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "h.30.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin",
+    "h.30.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin",
+    "h.30.self_attention.dense.weight": "pytorch_model-00002-of-00002.bin",
+    "h.30.self_attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin",
+    "h.31.input_layernorm.bias": "pytorch_model-00002-of-00002.bin",
+    "h.31.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "h.31.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00002.bin",
+    "h.31.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00002.bin",
+    "h.31.self_attention.dense.weight": "pytorch_model-00002-of-00002.bin",
+    "h.31.self_attention.query_key_value.weight": "pytorch_model-00002-of-00002.bin",
+    "h.4.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.4.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.4.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.4.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.4.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.4.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.5.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.5.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.5.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.5.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.5.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.5.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.6.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.6.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.6.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.6.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.6.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.6.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.7.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.7.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.7.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.7.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.7.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.7.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.8.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.8.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.8.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.8.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.8.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.8.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "h.9.input_layernorm.bias": "pytorch_model-00001-of-00002.bin",
+    "h.9.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "h.9.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.9.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00002.bin",
+    "h.9.self_attention.dense.weight": "pytorch_model-00001-of-00002.bin",
+    "h.9.self_attention.query_key_value.weight": "pytorch_model-00001-of-00002.bin",
+    "ln_f.bias": "pytorch_model-00002-of-00002.bin",
+    "ln_f.weight": "pytorch_model-00002-of-00002.bin",
+    "word_embeddings.weight": "pytorch_model-00001-of-00002.bin"
+  }
+}