File size: 1,851 Bytes
d71feba
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
from transformers import GPT2Config


class NomicBertConfig(GPT2Config):
    model_type = "nomic_bert"

    def __init__(self,
        prenorm=False,
        parallel_block=False,
        parallel_block_tied_norm=False,
        rotary_emb_fraction=0.0,
        fused_dropout_add_ln=False,
        fused_bias_fc=False,
        use_flash_attn=False,
        use_xentropy=False,
        qkv_proj_bias=True,
        rotary_emb_base=1000,
        rotary_emb_scale_base=None,
        rotary_emb_interleaved=False,
        mlp_fc1_bias=True,
        mlp_fc2_bias=True,
        use_rms_norm=False,
        causal=False,
        type_vocab_size=2,
        dense_seq_output=True,
        pad_vocab_size_multiple=1,
        tie_word_embeddings=True,
        rotary_scaling_factor=1.0,
        **kwargs,
    ):
        self.prenorm = prenorm
        self.parallel_block = parallel_block
        self.parallel_block_tied_norm = parallel_block_tied_norm
        self.rotary_emb_fraction = rotary_emb_fraction
        self.tie_word_embeddings = tie_word_embeddings
        self.fused_dropout_add_ln = fused_dropout_add_ln
        self.fused_bias_fc = fused_bias_fc
        self.use_flash_attn = use_flash_attn
        self.use_xentropy = use_xentropy
        self.qkv_proj_bias = qkv_proj_bias
        self.rotary_emb_base = rotary_emb_base
        self.rotary_emb_scale_base = rotary_emb_scale_base
        self.rotary_emb_interleaved = rotary_emb_interleaved
        self.mlp_fc1_bias = mlp_fc1_bias
        self.mlp_fc2_bias = mlp_fc2_bias
        self.use_rms_norm = use_rms_norm
        self.causal = causal
        self.type_vocab_size = type_vocab_size
        self.dense_seq_output = dense_seq_output
        self.pad_vocab_size_multiple = pad_vocab_size_multiple
        self.rotary_scaling_factor = rotary_scaling_factor

        super().__init__(**kwargs)