{ "_name_or_path": "/finetune-data/models/CarperAI/pythia-6.9b-deduped-4k", "architectures": [ "GPTNeoXForCausalLM" ], "bos_token_id": 0, "eos_token_id": 0, "gradient_checkpointing": true, "hidden_act": "gelu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 16384, "layer_norm_eps": 1e-05, "max_position_embeddings": 4096, "model_type": "gpt_neox", "num_attention_heads": 32, "num_hidden_layers": 32, "pad_token_id": 0, "rotary_emb_base": 10000, "rotary_pct": 0.25, "tie_word_embeddings": false, "torch_dtype": "float16", "transformers_version": "4.27.2", "use_cache": true, "use_parallel_residual": true, "vocab_size": 50277, "quantization_config": { "bits": 4, "group_size": 128, "damp_percent": 0.01, "desc_act": false, "sym": true, "true_sequential": true, "model_name_or_path": null, "model_file_base_name": "model", "quant_method": "gptq" } }