File size: 1,757 Bytes
8d36e55 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 |
from transformers import GPT2Config
class NomicBertConfig(GPT2Config):
model_type = "nomic_bert"
def __init__(self,
prenorm=False,
parallel_block=False,
parallel_block_tied_norm=False,
rotary_emb_fraction=0.0,
fused_dropout_add_ln=False,
fused_bias_fc=False,
use_flash_attn=False,
use_xentropy=False,
qkv_proj_bias=True,
rotary_emb_base=1000,
rotary_emb_scale_base=None,
rotary_emb_interleaved=False,
mlp_fc1_bias=True,
mlp_fc2_bias=True,
use_rms_norm=False,
causal=False,
type_vocab_size=2,
dense_seq_output=True,
pad_vocab_size_multiple=1,
tie_word_embeddings=True,
**kwargs,
):
self.prenorm = prenorm
self.parallel_block = parallel_block
self.parallel_block_tied_norm = parallel_block_tied_norm
self.rotary_emb_fraction = rotary_emb_fraction
self.tie_word_embeddings = tie_word_embeddings
self.fused_dropout_add_ln = fused_dropout_add_ln
self.fused_bias_fc = fused_bias_fc
self.use_flash_attn = use_flash_attn
self.use_xentropy = use_xentropy
self.qkv_proj_bias = qkv_proj_bias
self.rotary_emb_base = rotary_emb_base
self.rotary_emb_scale_base = rotary_emb_scale_base
self.rotary_emb_interleaved = rotary_emb_interleaved
self.mlp_fc1_bias = mlp_fc1_bias
self.mlp_fc2_bias = mlp_fc2_bias
self.use_rms_norm = use_rms_norm
self.causal = causal
self.type_vocab_size = type_vocab_size
self.dense_seq_output = dense_seq_output
self.pad_vocab_size_multiple = pad_vocab_size_multiple
super().__init__(**kwargs) |