from transformers import PretrainedConfig


class GEBConfig(PretrainedConfig):
    model_type = "geblm"
    def __init__(
        self,
        num_layers=24,
        padded_vocab_size=64896,
        hidden_size=2048,
        ffn_hidden_size=5632,
        kv_channels=128,
        num_attention_heads=16,
        torch_dtype='bfloat16',
        seq_length=4096,
        hidden_dropout=0.0,
        attention_dropout=0.0,
        layernorm_epsilon=1e-5,
        max_position_embeddings=4096,
        bias_dropout_fusion=True,
        use_cache=True,
        apply_residual_connection_post_layernorm=False,
        post_layer_norm=True,
        add_bias_linear=False,
        use_flash_attn=True,
        num_key_value_heads=4,
        apply_query_key_layer_scaling=False,
        attention_softmax_in_fp32=False,
        fp32_residual_connection=False,
        pre_seq_len=None,
        prefix_projection=False,
        tie_word_embeddings=False,
        **kwargs
    ):
        self.num_layers=num_layers
        self.padded_vocab_size=padded_vocab_size
        self.hidden_size=hidden_size
        self.ffn_hidden_size=ffn_hidden_size
        self.kv_channels=kv_channels
        self.num_attention_heads=num_attention_heads
        self.torch_dtype=torch_dtype
        self.seq_length=seq_length
        self.hidden_dropout=hidden_dropout,
        self.attention_dropout=attention_dropout
        self.layernorm_epsilon=layernorm_epsilon
        self.max_position_embeddings=max_position_embeddings
        self.bias_dropout_fusion=bias_dropout_fusion
        self.use_cache=use_cache
        self.apply_residual_connection_post_layernorm=apply_residual_connection_post_layernorm
        self.post_layer_norm=post_layer_norm
        self.add_bias_linear=add_bias_linear
        self.use_flash_attn=use_flash_attn
        self.num_key_value_heads=num_key_value_heads
        self.apply_query_key_layer_scaling=apply_query_key_layer_scaling
        self.attention_softmax_in_fp32=attention_softmax_in_fp32
        self.fp32_residual_connection=fp32_residual_connection
        self.pre_seq_len=pre_seq_len
        self.prefix_projection=prefix_projection
        self.tie_word_embeddings=tie_word_embeddings
        super().__init__(**kwargs)