Upload GPTRefactForCausalLM

configuration_gpt_refact.py

@@ -17,25 +17,24 @@ class GPTRefactConfig(PretrainedConfig):
 
     def __init__(
         self,
-        vocab_size=49216,
-        n_positions=1024,
-        n_embd=768,
-        n_layer=12,
-        n_head=12,
-        n_inner=None,
-        resid_pdrop=0.1,
-        embd_pdrop=0.1,
-        attn_pdrop=0.1,
+        vocab_size: int = 49216,
+        n_positions: int = 4096,
+        n_embd: int = 1024,
+        n_layer: int = 32,
+        n_head: int = 64,
+        max_position_embeddings: int = 4096,
+        multi_query: bool = True,
         layer_norm_epsilon=1e-5,
         initializer_range=0.02,
         scale_attn_weights=True,
         use_cache=True,
         bos_token_id=-1,
         eos_token_id=0,
-        max_position_embeddings: int = 4096,
-        multi_query: bool = True,
         attention_softmax_in_fp32=False,
         scale_attention_softmax_in_fp32=False,
+        resid_pdrop=0.1,
+        embd_pdrop=0.1,
+        attn_pdrop=0.1,
         **kwargs,
     ):
         self.vocab_size = vocab_size
@@ -43,7 +42,7 @@ class GPTRefactConfig(PretrainedConfig):
         self.n_embd = n_embd
         self.n_layer = n_layer
         self.n_head = n_head
-        self.n_inner = n_inner
+        self.n_inner = None
         self.resid_pdrop = resid_pdrop
         self.embd_pdrop = embd_pdrop
         self.attn_pdrop = attn_pdrop

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:81388e4a168bb437a7a09af6c8b6c2943990276ee62c2f449cd2bdff257e8860
+oid sha256:eb422076b1a52027d21772a7c4cbb7365078c2dd489384a97d6ce61c8b7b7204
 size 6343461637