|
{ |
|
"latent_size":256, |
|
"seed": 42, |
|
"ratio_increase": 0.5, |
|
"ratio_zero": 0.05, |
|
"dim_target_kl": 1.0, |
|
"length_weighted_loss": true, |
|
"beta_start": 0.0001, |
|
"beta_m": 0.1, |
|
"lambda_adv": 0.0, |
|
"fb_mode": 1, |
|
"mlm_probability": 0.15, |
|
"drop_prob": 0.0, |
|
"sub_prob": 0.0, |
|
"shuffle_dist": 0, |
|
"zeta": 0.0, |
|
"learning_rate": 5e-05, |
|
"weight_decay": 0.1, |
|
"adam_epsilon": 1e-08, |
|
"max_grad_norm": 1.0, |
|
"warmup": 0.01, |
|
"temperature": 1.0, |
|
"top_k": 0, |
|
"repetition_penalty": 2.0, |
|
"top_p": 0.9, |
|
"std_scale": 1.5, |
|
"max_out_length": 100, |
|
"logger": true, |
|
"checkpoint_callback": null, |
|
"enable_checkpointing": true, |
|
"process_position": 0, |
|
"encoder":{ |
|
"architectures": [ |
|
"BertForMaskedLM" |
|
], |
|
"attention_probs_dropout_prob": 0.1, |
|
"directionality": "bidi", |
|
"hidden_act": "gelu", |
|
"hidden_dropout_prob": 0.1, |
|
"hidden_size": 768, |
|
"initializer_range": 0.02, |
|
"intermediate_size": 3072, |
|
"layer_norm_eps": 1e-12, |
|
"max_position_embeddings": 512, |
|
"model_type": "bert", |
|
"num_attention_heads": 12, |
|
"num_hidden_layers": 12, |
|
"pad_token_id": 0, |
|
"pooler_fc_size": 768, |
|
"pooler_num_attention_heads": 12, |
|
"pooler_num_fc_layers": 3, |
|
"pooler_size_per_head": 128, |
|
"pooler_type": "first_token_transform", |
|
"type_vocab_size": 2, |
|
"vocab_size": 21128 |
|
}, |
|
"decoder":{ |
|
"num_layers":32, |
|
"vocab_size":50048, |
|
"hidden_size":1600, |
|
"num_attention_heads":25, |
|
"embedding_dropout_prob":0.1, |
|
"attention_dropout_prob":0.1, |
|
"output_dropout_prob":0.1, |
|
"max_sequence_length":512, |
|
"max_memory_length":512, |
|
"latent_size":256, |
|
"checkpoint_activations":false, |
|
"checkpoint_num_layers":1, |
|
"parallel_output":true, |
|
"relative_encoding":true |
|
} |
|
} |