| # @package _group_ | |
| common: | |
| tpu: true | |
| fp16: false | |
| log_format: json | |
| log_interval: 10 | |
| wandb_project: pretrain_tpu_960h | |
| checkpoint: | |
| save_interval_updates: 500 | |
| keep_interval_updates: 2 | |
| no_epoch_checkpoints: false | |
| task: | |
| _name: audio_pretraining | |
| data: ??? | |
| max_sample_size: 257600 | |
| min_sample_size: 46400 | |
| normalize: false | |
| num_batch_buckets: 3 | |
| precompute_mask_indices: true | |
| enable_padding: true | |
| dataset: | |
| num_workers: 12 | |
| max_tokens: 1200000 | |
| skip_invalid_size_inputs_valid_test: true | |
| distributed_training: | |
| distributed_world_size: 8 | |
| ddp_backend: legacy_ddp | |
| criterion: | |
| _name: wav2vec | |
| infonce: true | |
| log_keys: ["prob_perplexity","code_perplexity","temp"] | |
| loss_weights: [0.1, 0] | |
| optimization: | |
| max_update: 400000 | |
| lr: [0.0005] | |
| optimizer: | |
| _name: adam | |
| adam_betas: (0.9,0.98) | |
| adam_eps: 1e-06 | |
| weight_decay: 0.01 | |
| lr_scheduler: | |
| _name: polynomial_decay | |
| warmup_updates: 32000 | |
| model: | |
| _name: wav2vec2 | |
| quantize_targets: true | |
| final_dim: 256 | |
| encoder_layerdrop: 0.05 | |
| dropout_input: 0.1 | |
| dropout_features: 0.1 | |
| feature_grad_mult: 0.1 | |
| encoder_embed_dim: 768 | |