taufeeque's picture
Update config.json
917144d
{
"architectures": [
"GPTNeoXCodebookModel"
],
"codebook_at": [
"attn_preproj",
"mlp"
],
"codebook_kwargs": {},
"codebook_type": [
"group",
"vanilla"
],
"k_codebook": [
1,
1
],
"kmeans_init": false,
"kmeans_init_examples": 1000,
"kmeans_kwargs": null,
"kmeans_path": null,
"layers_to_snap": [
0,
1,
2,
3
],
"loss": "aeloss",
"model_type": "codebook",
"num_codebooks": [
4,
1
],
"num_codes": 10000,
"replace_codes": false,
"similarity_metric": "inner_product",
"torch_dtype": "float32",
"transformers_version": "4.28.1"
}