{ | |
"_name_or_path": "c:\\Users\\Filip\\Documents\\School\\Delft\\Research project\\results\\upload\\GPT-PReLU", | |
"activation_function": "gelu_new", | |
"architectures": [ | |
"ActivationsGPTNeoForCausalLM" | |
], | |
"attention_dropout": 0.0, | |
"attention_layers": [ | |
"global", | |
"local" | |
], | |
"attention_types": [ | |
[ | |
[ | |
"global", | |
"local" | |
], | |
1 | |
] | |
], | |
"bos_token_id": 50256, | |
"classifier_dropout": 0.1, | |
"custom_activation": "prelu", | |
"embed_dropout": 0.0, | |
"eos_token_id": 50256, | |
"hidden_size": 512, | |
"initializer_range": 0.02, | |
"intermediate_size": 1024, | |
"layer_norm_epsilon": 1e-05, | |
"max_position_embeddings": 512, | |
"model_type": "activations_gpt_neo", | |
"num_heads": 4, | |
"num_layers": 2, | |
"pad_token_id": 0, | |
"resid_dropout": 0.0, | |
"torch_dtype": "float32", | |
"transformers_version": "4.26.1", | |
"use_cache": true, | |
"vocab_size": 10000, | |
"window_size": 256 | |
} | |