def run_neox_args_load_test(yaml_files):
    from megatron.neox_arguments import NeoXArgs # [STATE] NeoXArgs = <class 'megatron.neox_arguments.arguments.NeoXArgs'> [/STATE]

    yaml_list = get_configs_with_path(yaml_files) # [STATE] yaml_list = ['/local/rcs/XXX/code/pytrace-collector/logs/self_collected/tried/EleutherAI+gpt-neox/EleutherAI+gpt-neox/configs/125M.yml', '/local/rcs/XXX/code/pytrace-collector/logs/self_collected/tried/EleutherAI+gpt-neox/EleutherAI+gpt-neox/configs/local_setup.yml', '/local/rcs/XXX/code/pytrace-collector/logs/self_collected/tried/EleutherAI+gpt-neox/EleutherAI+gpt-neox/configs/cpu_mock_config.yml'] [/STATE]
    args_loaded = NeoXArgs.from_ymls(yaml_list) # [STATE] args_loaded = NeoXArgs(distributed_backend='nccl', local_rank=None, rank=None, lazy_mpu_init=False, short_seq_prob=0.1, eod_mask_loss=False, adlr_autoresume=False, adlr_autoresume_interval=1000, seed=1234, onnx_safe=False, deepscale=False, deepscale_config=None, deepspeed_mpi=False, deepspeed_slurm=False, user_script=None, iteration=None, do_train=None, do_valid=None, do_test=None, save_iters=[10000, 20000, 30000, 40000, 50000, 60000, 70000, 80000, 90000, 100000, 110000, 120000, 130000, 140000, 150000, 160000, 170000, 180000, 190000, 200000, 210000, 220000, 230000, 240000, 250000, 260000, 270000, 280000, 290000, 300000, 310000], global_num_gpus=1, text_gen_type='unconditional', temperature=0.0, top_p=0.0, top_k=0, return_logits=False, maximum_tokens=64, prompt_end='\n', sample_input_file=None, sample_output_file='samples.txt', num_samples=1, recompute=False, eval_results_prefix='', eval_tasks=None, use_wandb=True, wandb_group=None, wandb_team=None, wandb_project='neox', wandb_host='https://api.wandb.ai', wandb_init_all_ranks=False, git_hash='7a8fa2f0', log_dir='logs', tensorboard_dir='tensorboard', log_interval=100, log_grad_pct_zeros=False, log_param_norm=False, log_grad_norm=False, log_optimizer_states=False, log_gradient_noise_scale=False, gradient_noise_scale_n_batches=5, gradient_noise_scale_cpu_offload=False, pipe_parallel_size=1, model_parallel_size=1, pipe_partition_method='type:transformer|mlp', world_size=None, is_pipe_parallel=True, data_path='data/enwik8/enwik8_text_document', use_shared_fs=True, train_data_paths=None, label_data_paths=None, test_data_paths=None, valid_data_paths=None, train_data_weights=None, valid_data_weights=None, test_data_weights=None, weight_by_num_documents=False, weighted_sampler_alpha=1.0, data_impl='mmap', mmap_warmup=False, save='checkpoints', s3_path=None, s3_chunk_size=104857600, config_files={'125M.yml': '# GPT-2 pretraining setup\n{\n   # parallelism settings ( you will want to change these based on your cluster setup, ideally scheduling pipeline stages\n   # across the node boundaries )\n   "pipe_parallel_size": 1,\n   "model_parallel_size": 1,\n\n   # model settings\n   "num_layers": 12,\n   "hidden_size": 768,\n   "num_attention_heads": 12,\n   "seq_length": 2048,\n   "max_position_embeddings": 2048,\n   "norm": "layernorm",\n   "pos_emb": "rotary",\n   "no_weight_tying": true,\n   "gpt_j_residual": false,\n   "output_layer_parallelism": "column",\n\n   # these should provide some speedup but takes a while to build, set to true if desired\n   "scaled_upper_triang_masked_softmax_fusion": false,\n   "bias_gelu_fusion": false,\n   "rope_fusion": false,\n\n   # init methods\n   "init_method": "small_init",\n   "output_layer_init_method": "wang_init",\n\n\n   # optimizer settings\n   "optimizer": {\n     "type": "Adam",\n     "params": {\n       "lr": 0.0006,\n       "betas": [0.9, 0.95],\n       "eps": 1.0e-8,\n     }\n   },\n   "min_lr": 0.00006,\n\n   # for all zero_optimization options, see https://www.deepspeed.ai/docs/config-json/#zero-optimizations-for-fp16-training\n   "zero_optimization": {\n    "stage": 1,\n    "allgather_partitions": True,\n    "allgather_bucket_size": 500000000,\n    "overlap_comm": True,\n    "reduce_scatter": True,\n    "reduce_bucket_size": 500000000,\n    "contiguous_gradients": True,\n  },\n\n   # batch / data settings\n   "train_micro_batch_size_per_gpu": 4,\n   "data_impl": "mmap",\n\n   # activation checkpointing\n   "checkpoint_activations": true,\n   "checkpoint_num_layers": 1,\n   "partition_activations": true,\n   "synchronize_each_layer": true,\n\n   # regularization\n   "gradient_clipping": 1.0,\n   "weight_decay": 0.1,\n   "hidden_dropout": 0.0,\n   "attention_dropout": 0.0,\n\n   # precision settings\n   "fp16": {\n     "enabled": true,\n     "loss_scale": 0,\n     "loss_scale_window": 1000,\n     "hysteresis": 2,\n     "min_loss_scale": 1\n   },\n\n   # misc. training settings\n   "train_iters": 320000,\n   "lr_decay_iters": 320000,\n   "distributed_backend": "nccl",\n   "lr_decay_style": "cosine",\n   "warmup": 0.01,\n   "checkpoint_factor": 10...\n{\n  "global_num_gpus": 1\n}\n'}, load='checkpoints', checkpoint_validation_with_forward_pass=False, checkpoint_scale='linear', checkpoint_factor=10000, extra_save_iters=None, no_save_optim=False, no_save_rng=False, no_load_optim=False, no_load_rng=False, finetune=False, batch_size=4, train_iters=320000, eval_iters=10, keep_last_n_checkpoints=4, eval_interval=1000, split='969, 30, 1', vocab_file='data/gpt2-vocab.json', merge_file='data/gpt2-merges.txt', num_workers=2, exit_interval=None, attention_dropout=0.0, hidden_dropout=0.0, weight_decay=0.1, checkpoint_activations=True, checkpoint_num_layers=1, deepspeed_activation_checkpointing=True, contiguous_checkpointing=False, checkpoint_in_cpu=False, synchronize_each_layer=True, profile_backward=False, partition_activations=True, gas=1, clip_grad=1.0, hysteresis=2, dynamic_loss_scale=True, loss_scale=None, loss_scale_window=1000.0, min_scale=1.0, char_level_ppl=False, use_mup=False, coord_check=False, save_base_shapes=False, base_shapes_file=None, mup_init_scale=1.0, mup_attn_temp=1.0, mup_output_temp=1.0, mup_embedding_mult=1.0, mup_rp_embedding_mult=1.0, mup_width_scale=2, tokenizer_type='GPT2BPETokenizer', padded_vocab_size=None, optimizer_type='Adam', use_bnb_optimizer=False, zero_stage=1, zero_reduce_scatter=True, zero_contiguous_gradients=True, zero_reduce_bucket_size=500000000, zero_allgather_bucket_size=500000000, lr=0.0006, lr_decay_style='cosine', lr_decay_iters=320000, min_lr=6e-05, warmup=0.01, override_lr_scheduler=False, use_checkpoint_lr_scheduler=False, precision='fp16', num_layers=12, hidden_size=768, num_attention_heads=12, seq_length=2048, max_position_embeddings=2048, norm='layernorm', use_qk_layernorm=False, layernorm_epsilon=1e-05, rms_norm_epsilon=1e-08, scalenorm_epsilon=1e-08, pos_emb='rotary', rpe_num_buckets=32, rpe_max_distance=128, opt_pos_emb_offset=0, no_weight_tying=True, attention_config=['global', 'global', 'global', 'global', 'global', 'global', 'global', 'global', 'global', 'global', 'global', 'global'], sparsity_config={}, num_unique_layers=None, param_sharing_style='grouped', make_vocab_size_divisible_by=128, activation='gelu', scaled_upper_triang_masked_softmax_fusion=False, scaled_masked_softmax_fusion=False, bias_gelu_fusion=False, bias_dropout_fusion=False, rope_fusion=False, fp16_lm_cross_entropy=False, init_method_std=0.02, apply_query_key_layer_scaling=False, use_cpu_initialization=False, attention_softmax_in_fp32=False, rotary_pct=1.0, rotary_emb_base=10000, init_method='small_init', output_layer_init_method='wang_init', gmlp_attn_dim=64, gpt_j_residual=False, gpt_j_tied=False, use_bias_in_norms=True, use_bias_in_attn_linear=True, mlp_type='regular', soft_prompt_tuning=None, output_layer_parallelism='column', deepspeed=True, train_batch_size=4, train_micro_batch_size_per_gpu=4, gradient_accumulation_steps=1, optimizer={'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, scheduler=None, fp32_allreduce=False, prescale_gradients=False, gradient_predivide_factor=1.0, sparse_gradients=False, fp16={'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, bf16=None, amp=None, gradient_clipping=1.0, zero_optimization={'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, curriculum_learning=None, curriculum_seqlen=0, steps_per_print=10, wall_clock_breakdown=True, dump_state=False, flops_profiler=None, communication_data_type=None, autotuning=None, activation_checkpointing=None, sparse_attention=None, data_efficiency=None, tensorboard=None, wandb=None, csv_monitor=None, elasticity=None, comms_logger=None, compression_training=None, checkpoint=None, data_types=None, deepspeed_extra_args=None, hostfile='/mock_path', include=None, exclude=None, num_nodes=-1, num_gpus=None, master_port=29500, master_addr=None, launcher='pdsh', force_multi=False, detect_nvlink_pairs=False, autotuning_run=None, no_ssh_check=False, comment=None, account=None) [/STATE]
    assert isinstance(args_loaded, NeoXArgs) # [STATE] @py_assert3 = None [/STATE]

    # initialize an empty config dictionary to be filled by yamls
    config = dict() # [STATE] config = {} [/STATE]

    # iterate of all to be loaded yaml files
    for conf_file_name in yaml_list: #  [STATE] 3 [/STATE]

        # load file
        with open(conf_file_name) as conf_file: #  [STATE] conf_file = <_io.TextIOWrapper name='/local/rcs/XXX/code/pytrace-collector/logs/self_collected/tried/EleutherAI+gpt-neox/EleutherAI+gpt-neox/configs/125M.yml' mode='r' encoding='UTF-8'> [/STATE] [STATE] conf_file = <_io.TextIOWrapper name='/local/rcs/XXX/code/pytrace-collector/logs/self_collected/tried/EleutherAI+gpt-neox/EleutherAI+gpt-neox/configs/local_setup.yml' mode='r' encoding='UTF-8'> [/STATE] [STATE] conf_file = <_io.TextIOWrapper name='/local/rcs/XXX/code/pytrace-collector/logs/self_collected/tried/EleutherAI+gpt-neox/EleutherAI+gpt-neox/configs/cpu_mock_config.yml' mode='r' encoding='UTF-8'> [/STATE]
            conf = yaml.load(conf_file, Loader=yaml.FullLoader) #  [STATE] conf = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01, 'checkpoint_factor': 10000, 'eval_interval': 1000, 'eval_iters': 10, 'log_interval': 100, 'steps_per_print': 10, 'keep_last_n_checkpoints': 4, 'wall_clock_breakdown': True, 'hostfile': '/mock_path'} [/STATE] [STATE] conf = {'data_path': 'data/enwik8/enwik8_text_document', 'vocab_file': 'data/gpt2-vocab.json', 'merge_file': 'data/gpt2-merges.txt', 'save': 'checkpoints', 'load': 'checkpoints', 'checkpoint_validation_with_forward_pass': False, 'tensorboard_dir': 'tensorboard', 'log_dir': 'logs', 'use_wandb': True, 'wandb_host': 'https://api.wandb.ai', 'wandb_project': 'neox'} [/STATE] [STATE] conf = {'global_num_gpus': 1} [/STATE]

        # check for key duplicates and load values
        for conf_key, conf_value in conf.items(): #  [STATE] 58 [/STATE]
            if conf_key in config:
                raise ValueError(
                    f"Conf file {conf_file_name} has the following duplicate keys with previously loaded file: {conf_key}"
                )

            conf_key_converted = conf_key.replace( #  [STATE] conf_key_converted = 'pipe_parallel_size' [/STATE] [STATE] conf_key_converted = 'model_parallel_size' [/STATE] [STATE] conf_key_converted = 'num_layers' [/STATE] [STATE] conf_key_converted = 'hidden_size' [/STATE] [STATE] conf_key_converted = 'num_attention_heads' [/STATE] [STATE] conf_key_converted = 'seq_length' [/STATE] [STATE] conf_key_converted = 'max_position_embeddings' [/STATE] [STATE] conf_key_converted = 'norm' [/STATE] [STATE] conf_key_converted = 'pos_emb' [/STATE] [STATE] conf_key_converted = 'no_weight_tying' [/STATE] [STATE] conf_key_converted = 'gpt_j_residual' [/STATE] [STATE] conf_key_converted = 'output_layer_parallelism' [/STATE] [STATE] conf_key_converted = 'scaled_upper_triang_masked_softmax_fusion' [/STATE] [STATE] conf_key_converted = 'bias_gelu_fusion' [/STATE] [STATE] conf_key_converted = 'rope_fusion' [/STATE] [STATE] conf_key_converted = 'init_method' [/STATE] [STATE] conf_key_converted = 'output_layer_init_method' [/STATE] [STATE] conf_key_converted = 'optimizer' [/STATE] [STATE] conf_key_converted = 'min_lr' [/STATE] [STATE] conf_key_converted = 'zero_optimization' [/STATE] [STATE] conf_key_converted = 'train_micro_batch_size_per_gpu' [/STATE] [STATE] conf_key_converted = 'data_impl' [/STATE] [STATE] conf_key_converted = 'checkpoint_activations' [/STATE] [STATE] conf_key_converted = 'checkpoint_num_layers' [/STATE] [STATE] conf_key_converted = 'partition_activations' [/STATE] [STATE] conf_key_converted = 'synchronize_each_layer' [/STATE] [STATE] conf_key_converted = 'gradient_clipping' [/STATE] [STATE] conf_key_converted = 'weight_decay' [/STATE] [STATE] conf_key_converted = 'hidden_dropout' [/STATE] [STATE] conf_key_converted = 'attention_dropout' [/STATE] [STATE] conf_key_converted = 'fp16' [/STATE] [STATE] conf_key_converted = 'train_iters' [/STATE] [STATE] conf_key_converted = 'lr_decay_iters' [/STATE] [STATE] conf_key_converted = 'distributed_backend' [/STATE] [STATE] conf_key_converted = 'lr_decay_style' [/STATE] [STATE] conf_key_converted = 'warmup' [/STATE] [STATE] conf_key_converted = 'checkpoint_factor' [/STATE] [STATE] conf_key_converted = 'eval_interval' [/STATE] [STATE] conf_key_converted = 'eval_iters' [/STATE] [STATE] conf_key_converted = 'log_interval' [/STATE] [STATE] conf_key_converted = 'steps_per_print' [/STATE] [STATE] conf_key_converted = 'keep_last_n_checkpoints' [/STATE] [STATE] conf_key_converted = 'wall_clock_breakdown' [/STATE] [STATE] conf_key_converted = 'hostfile' [/STATE] [STATE] conf_key_converted = 'data_path' [/STATE] [STATE] conf_key_converted = 'vocab_file' [/STATE] [STATE] conf_key_converted = 'merge_file' [/STATE] [STATE] conf_key_converted = 'save' [/STATE] [STATE] conf_key_converted = 'load' [/STATE] [STATE] conf_key_converted = 'checkpoint_validation_with_forward_pass' [/STATE] [STATE] conf_key_converted = 'tensorboard_dir' [/STATE] [STATE] conf_key_converted = 'log_dir' [/STATE] [STATE] conf_key_converted = 'use_wandb' [/STATE] [STATE] conf_key_converted = 'wandb_host' [/STATE] [STATE] conf_key_converted = 'wandb_project' [/STATE] [STATE] conf_key_converted = 'global_num_gpus' [/STATE]
                "-", "_"
            )  # TODO remove replace and update configuration files?
            config[conf_key_converted] = conf_value #  [STATE] config = {'pipe_parallel_size': 1} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm'} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary'} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column'} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init'} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init'} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap'} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl'} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine'} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01, 'checkpoint_factor': 10000} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01, 'checkpoint_factor': 10000, 'eval_interval': 1000} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01, 'checkpoint_factor': 10000, 'eval_interval': 1000, 'eval_iters': 10} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01, 'checkpoint_factor': 10000, 'eval_interval': 1000, 'eval_iters': 10, 'log_interval': 100} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01, 'checkpoint_factor': 10000, 'eval_interval': 1000, 'eval_iters': 10, 'log_interval': 100, 'steps_per_print': 10} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01, 'checkpoint_factor': 10000, 'eval_interval': 1000, 'eval_iters': 10, 'log_interval': 100, 'steps_per_print': 10, 'keep_last_n_checkpoints': 4} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01, 'checkpoint_factor': 10000, 'eval_interval': 1000, 'eval_iters': 10, 'log_interval': 100, 'steps_per_print': 10, 'keep_last_n_checkpoints': 4, 'wall_clock_breakdown': True} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01, 'checkpoint_factor': 10000, 'eval_interval': 1000, 'eval_iters': 10, 'log_interval': 100, 'steps_per_print': 10, 'keep_last_n_checkpoints': 4, 'wall_clock_breakdown': True, 'hostfile': '/mock_path'} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01, 'checkpoint_factor': 10000, 'eval_interval': 1000, 'eval_iters': 10, 'log_interval': 100, 'steps_per_print': 10, 'keep_last_n_checkpoints': 4, 'wall_clock_breakdown': True, 'hostfile': '/mock_path', 'data_path': 'data/enwik8/enwik8_text_document'} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01, 'checkpoint_factor': 10000, 'eval_interval': 1000, 'eval_iters': 10, 'log_interval': 100, 'steps_per_print': 10, 'keep_last_n_checkpoints': 4, 'wall_clock_breakdown': True, 'hostfile': '/mock_path', 'data_path': 'data/enwik8/enwik8_text_document', 'vocab_file': 'data/gpt2-vocab.json'} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01, 'checkpoint_factor': 10000, 'eval_interval': 1000, 'eval_iters': 10, 'log_interval': 100, 'steps_per_print': 10, 'keep_last_n_checkpoints': 4, 'wall_clock_breakdown': True, 'hostfile': '/mock_path', 'data_path': 'data/enwik8/enwik8_text_document', 'vocab_file': 'data/gpt2-vocab.json', 'merge_file': 'data/gpt2-merges.txt'} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01, 'checkpoint_factor': 10000, 'eval_interval': 1000, 'eval_iters': 10, 'log_interval': 100, 'steps_per_print': 10, 'keep_last_n_checkpoints': 4, 'wall_clock_breakdown': True, 'hostfile': '/mock_path', 'data_path': 'data/enwik8/enwik8_text_document', 'vocab_file': 'data/gpt2-vocab.json', 'merge_file': 'data/gpt2-merges.txt', 'save': 'checkpoints'} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01, 'checkpoint_factor': 10000, 'eval_interval': 1000, 'eval_iters': 10, 'log_interval': 100, 'steps_per_print': 10, 'keep_last_n_checkpoints': 4, 'wall_clock_breakdown': True, 'hostfile': '/mock_path', 'data_path': 'data/enwik8/enwik8_text_document', 'vocab_file': 'data/gpt2-vocab.json', 'merge_file': 'data/gpt2-merges.txt', 'save': 'checkpoints', 'load': 'checkpoints'} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01, 'checkpoint_factor': 10000, 'eval_interval': 1000, 'eval_iters': 10, 'log_interval': 100, 'steps_per_print': 10, 'keep_last_n_checkpoints': 4, 'wall_clock_breakdown': True, 'hostfile': '/mock_path', 'data_path': 'data/enwik8/enwik8_text_document', 'vocab_file': 'data/gpt2-vocab.json', 'merge_file': 'data/gpt2-merges.txt', 'save': 'checkpoints', 'load': 'checkpoints', 'checkpoint_validation_with_forward_pass': False} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01, 'checkpoint_factor': 10000, 'eval_interval': 1000, 'eval_iters': 10, 'log_interval': 100, 'steps_per_print': 10, 'keep_last_n_checkpoints': 4, 'wall_clock_breakdown': True, 'hostfile': '/mock_path', 'data_path': 'data/enwik8/enwik8_text_document', 'vocab_file': 'data/gpt2-vocab.json', 'merge_file': 'data/gpt2-merges.txt', 'save': 'checkpoints', 'load': 'checkpoints', 'checkpoint_validation_with_forward_pass': False, 'tensorboard_dir': 'tensorboard'} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01, 'checkpoint_factor': 10000, 'eval_interval': 1000, 'eval_iters': 10, 'log_interval': 100, 'steps_per_print': 10, 'keep_last_n_checkpoints': 4, 'wall_clock_breakdown': True, 'hostfile': '/mock_path', 'data_path': 'data/enwik8/enwik8_text_document', 'vocab_file': 'data/gpt2-vocab.json', 'merge_file': 'data/gpt2-merges.txt', 'save': 'checkpoints', 'load': 'checkpoints', 'checkpoint_validation_with_forward_pass': False, 'tensorboard_dir': 'tensorboard', 'log_dir': 'logs'} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01, 'checkpoint_factor': 10000, 'eval_interval': 1000, 'eval_iters': 10, 'log_interval': 100, 'steps_per_print': 10, 'keep_last_n_checkpoints': 4, 'wall_clock_breakdown': True, 'hostfile': '/mock_path', 'data_path': 'data/enwik8/enwik8_text_document', 'vocab_file': 'data/gpt2-vocab.json', 'merge_file': 'data/gpt2-merges.txt', 'save': 'checkpoints', 'load': 'checkpoints', 'checkpoint_validation_with_forward_pass': False, 'tensorboard_dir': 'tensorboard', 'log_dir': 'logs', 'use_wandb': True} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01, 'checkpoint_factor': 10000, 'eval_interval': 1000, 'eval_iters': 10, 'log_interval': 100, 'steps_per_print': 10, 'keep_last_n_checkpoints': 4, 'wall_clock_breakdown': True, 'hostfile': '/mock_path', 'data_path': 'data/enwik8/enwik8_text_document', 'vocab_file': 'data/gpt2-vocab.json', 'merge_file': 'data/gpt2-merges.txt', 'save': 'checkpoints', 'load': 'checkpoints', 'checkpoint_validation_with_forward_pass': False, 'tensorboard_dir': 'tensorboard', 'log_dir': 'logs', 'use_wandb': True, 'wandb_host': 'https://api.wandb.ai'} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01, 'checkpoint_factor': 10000, 'eval_interval': 1000, 'eval_iters': 10, 'log_interval': 100, 'steps_per_print': 10, 'keep_last_n_checkpoints': 4, 'wall_clock_breakdown': True, 'hostfile': '/mock_path', 'data_path': 'data/enwik8/enwik8_text_document', 'vocab_file': 'data/gpt2-vocab.json', 'merge_file': 'data/gpt2-merges.txt', 'save': 'checkpoints', 'load': 'checkpoints', 'checkpoint_validation_with_forward_pass': False, 'tensorboard_dir': 'tensorboard', 'log_dir': 'logs', 'use_wandb': True, 'wandb_host': 'https://api.wandb.ai', 'wandb_project': 'neox'} [/STATE] [STATE] config = {'pipe_parallel_size': 1, 'model_parallel_size': 1, 'num_layers': 12, 'hidden_size': 768, 'num_attention_heads': 12, 'seq_length': 2048, 'max_position_embeddings': 2048, 'norm': 'layernorm', 'pos_emb': 'rotary', 'no_weight_tying': True, 'gpt_j_residual': False, 'output_layer_parallelism': 'column', 'scaled_upper_triang_masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'rope_fusion': False, 'init_method': 'small_init', 'output_layer_init_method': 'wang_init', 'optimizer': {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}}, 'min_lr': 6e-05, 'zero_optimization': {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True}, 'train_micro_batch_size_per_gpu': 4, 'data_impl': 'mmap', 'checkpoint_activations': True, 'checkpoint_num_layers': 1, 'partition_activations': True, 'synchronize_each_layer': True, 'gradient_clipping': 1.0, 'weight_decay': 0.1, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'fp16': {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1}, 'train_iters': 320000, 'lr_decay_iters': 320000, 'distributed_backend': 'nccl', 'lr_decay_style': 'cosine', 'warmup': 0.01, 'checkpoint_factor': 10000, 'eval_interval': 1000, 'eval_iters': 10, 'log_interval': 100, 'steps_per_print': 10, 'keep_last_n_checkpoints': 4, 'wall_clock_breakdown': True, 'hostfile': '/mock_path', 'data_path': 'data/enwik8/enwik8_text_document', 'vocab_file': 'data/gpt2-vocab.json', 'merge_file': 'data/gpt2-merges.txt', 'save': 'checkpoints', 'load': 'checkpoints', 'checkpoint_validation_with_forward_pass': False, 'tensorboard_dir': 'tensorboard', 'log_dir': 'logs', 'use_wandb': True, 'wandb_host': 'https://api.wandb.ai', 'wandb_project': 'neox', 'global_num_gpus': 1} [/STATE]

    # validate that neox args has the same value as specified in the config (if specified in the config)
    for k, v in config.items(): #  [STATE] 56 [/STATE]
        neox_args_value = getattr(args_loaded, k) #  [STATE] neox_args_value = 1 [/STATE] [STATE] neox_args_value = 12 [/STATE] [STATE] neox_args_value = 768 [/STATE] [STATE] neox_args_value = 2048 [/STATE] [STATE] neox_args_value = 'layernorm' [/STATE] [STATE] neox_args_value = 'rotary' [/STATE] [STATE] neox_args_value = True [/STATE] [STATE] neox_args_value = False [/STATE] [STATE] neox_args_value = 'column' [/STATE] [STATE] neox_args_value = 'small_init' [/STATE] [STATE] neox_args_value = 'wang_init' [/STATE] [STATE] neox_args_value = {'type': 'Adam', 'params': {'lr': 0.0006, 'betas': [0.9, 0.95], 'eps': 1e-08}} [/STATE] [STATE] neox_args_value = 6e-05 [/STATE] [STATE] neox_args_value = {'stage': 1, 'allgather_partitions': True, 'allgather_bucket_size': 500000000, 'overlap_comm': True, 'reduce_scatter': True, 'reduce_bucket_size': 500000000, 'contiguous_gradients': True} [/STATE] [STATE] neox_args_value = 4 [/STATE] [STATE] neox_args_value = 'mmap' [/STATE] [STATE] neox_args_value = 1.0 [/STATE] [STATE] neox_args_value = 0.1 [/STATE] [STATE] neox_args_value = 0.0 [/STATE] [STATE] neox_args_value = {'enabled': True, 'loss_scale': 0, 'loss_scale_window': 1000, 'hysteresis': 2, 'min_loss_scale': 1} [/STATE] [STATE] neox_args_value = 320000 [/STATE] [STATE] neox_args_value = 'nccl' [/STATE] [STATE] neox_args_value = 'cosine' [/STATE] [STATE] neox_args_value = 0.01 [/STATE] [STATE] neox_args_value = 10000 [/STATE] [STATE] neox_args_value = 1000 [/STATE] [STATE] neox_args_value = 10 [/STATE] [STATE] neox_args_value = 100 [/STATE] [STATE] neox_args_value = '/mock_path' [/STATE] [STATE] neox_args_value = 'data/enwik8/enwik8_text_document' [/STATE] [STATE] neox_args_value = 'data/gpt2-vocab.json' [/STATE] [STATE] neox_args_value = 'data/gpt2-merges.txt' [/STATE] [STATE] neox_args_value = 'checkpoints' [/STATE] [STATE] neox_args_value = 'tensorboard' [/STATE] [STATE] neox_args_value = 'logs' [/STATE] [STATE] neox_args_value = 'https://api.wandb.ai' [/STATE] [STATE] neox_args_value = 'neox' [/STATE]
        assert v == neox_args_value, ( #  [STATE] @py_assert1 = None [/STATE]
            "loaded neox args value "
            + str(k)
            + " == "
            + str(neox_args_value)
            + " different from config file "
            + str(v)
        )

run_neox_args_load_test(['125M.yml', 'local_setup.yml', 'cpu_mock_config.yml'])

1. def run_neox_args_load_test(yaml_files):
2.     from megatron.neox_arguments import NeoXArgs
3.
4.     yaml_list = get_configs_with_path(yaml_files)
5.     args_loaded = NeoXArgs.from_ymls(yaml_list)
6.     assert isinstance(args_loaded, NeoXArgs)
7.
8.     # initialize an empty config dictionary to be filled by yamls
9.     config = dict()
10.
11.     # iterate of all to be loaded yaml files
12.     for conf_file_name in yaml_list:
13.
14.         # load file
15.         with open(conf_file_name) as conf_file:
16.             conf = yaml.load(conf_file, Loader=yaml.FullLoader)
17.
18.         # check for key duplicates and load values
19.         for conf_key, conf_value in conf.items():
20.             if conf_key in config:
21.                 raise ValueError(
22.                     f"Conf file {conf_file_name} has the following duplicate keys with previously loaded file: {conf_key}"
23.                 )
24.
25.             conf_key_converted = conf_key.replace(
26.                 "-", "_"
27.             )  # TODO remove replace and update configuration files?
28.             config[conf_key_converted] = conf_value
29.
30.     # validate that neox args has the same value as specified in the config (if specified in the config)
31.     for k, v in config.items():
32.         neox_args_value = getattr(args_loaded, k)
33.         assert v == neox_args_value, (
34.             "loaded neox args value "
35.             + str(k)
36.             + " == "
37.             + str(neox_args_value)
38.             + " different from config file "
39.             + str(v)
40.         )
41.
42. run_neox_args_load_test(['125M.yml', 'local_setup.yml', 'cpu_mock_config.yml'])

Question: How many times will the loop on line 31 execute when 'run_neox_args_load_test(['125M.yml', 'local_setup.yml', 'cpu_mock_config.yml'])' is called?

Answer: 56


def main(input_args=None):
    args = get_args(input_args) # [STATE] args = Namespace(input='./tests/data/enwik8_first100.txt', jsonl_keys=['text'], num_docs=None, tokenizer_type='HFGPT2Tokenizer', vocab_file='gpt2', merge_file='./data/gpt2-merges.txt', append_eod=True, ftfy=False, output_prefix='./tests/data/enwik8_first100', dataset_impl='mmap', workers=1, log_interval=100, keep_empty=False, rank=0, make_vocab_size_divisible_by=128, model_parallel_size=1) [/STATE]
    encoder = Encoder(args) # [STATE] encoder = {args=Namespace(input='./tests/data/enwik8_first100.txt', jsonl_keys=['text'], num_docs=None, tokenizer_type='HFGPT2Tokenizer', vocab_file='gpt2', merge_file='./data/gpt2-merges.txt', append_eod=True, ftfy=False, output_prefix='./tests/data/enwik8_first100', dataset_impl='mmap', workers=1, log_interval=100, keep_empty=False, rank=0, make_vocab_size_divisible_by=128, model_parallel_size=1)} [/STATE]
    tokenizer = build_tokenizer(args) # [STATE] tokenizer = {name='HFGPT2TokenizerFast', tokenizer=GPT2TokenizerFast(name_or_path='gpt2', vocab_size=50257, model_max_length=1024, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'bos_token': '<|endoftext|>', 'eos_token': '<|endoftext|>', 'unk_token': '<|endoftext|>', 'pad_token': '<|padding|>'}, clean_up_tokenization_spaces=True),  added_tokens_decoder={	50256: AddedToken("<|endoftext|>", rstrip=False, lstrip=False, single_word=False, normalized=True, special=True),	50257: AddedToken("<|padding|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),}, eod_id=50256, pad_id=50257} [/STATE] # [STATE] args = Namespace(input='./tests/data/enwik8_first100.txt', jsonl_keys=['text'], num_docs=None, tokenizer_type='HFGPT2Tokenizer', vocab_file='gpt2', merge_file='./data/gpt2-merges.txt', append_eod=True, ftfy=False, output_prefix='./tests/data/enwik8_first100', dataset_impl='mmap', workers=1, log_interval=100, keep_empty=False, rank=0, make_vocab_size_divisible_by=128, model_parallel_size=1, padded_vocab_size=50304) [/STATE] # [STATE] encoder = {args=Namespace(input='./tests/data/enwik8_first100.txt', jsonl_keys=['text'], num_docs=None, tokenizer_type='HFGPT2Tokenizer', vocab_file='gpt2', merge_file='./data/gpt2-merges.txt', append_eod=True, ftfy=False, output_prefix='./tests/data/enwik8_first100', dataset_impl='mmap', workers=1, log_interval=100, keep_empty=False, rank=0, make_vocab_size_divisible_by=128, model_parallel_size=1, padded_vocab_size=50304)} [/STATE]
    print(f"Vocab size: {tokenizer.vocab_size}")
    print(f"Output prefix: {args.output_prefix}")

    # build a semaphore object to stop `yield_from_files` from getting ahead of encoder.encode and
    # hence building up memory
    semaphore = Semaphore(10000 + args.workers) # [STATE] semaphore = {_cond=<Condition(<unlocked _thread.lock object at 0x7f1a969bf450>, 0)>, _value=10001} [/STATE]

    # use multiprocessing to iterate over input documents
    fin = yield_from_files(args.input.split(","), semaphore) # [STATE] fin = <generator object yield_from_files at 0x7f1b94b985f0> [/STATE]

    if args.workers > 1:
        pool = multiprocessing.Pool(args.workers, initializer=encoder.initializer)
        encoded_docs = pool.imap(encoder.encode, fin, chunksize=25)
    else:
        encoder.initializer()
        encoded_docs = (encoder.encode(doc) for doc in fin) # [STATE] encoded_docs = <generator object main.<locals>.<genexpr> at 0x7f1a97c25270> [/STATE]

    # make a dataset builder for each key in args.jsonl_keys
    # each key will output to a different file beginning with args.output_prefix
    output_bin_files = {} # [STATE] output_bin_files = {} [/STATE]
    output_idx_files = {} # [STATE] output_idx_files = {} [/STATE]
    builders = {} # [STATE] builders = {} [/STATE]
    for key in args.jsonl_keys: #  [STATE] 1 [/STATE]
        output_bin_files[key] = "{}_{}_{}.bin".format( # [STATE] output_bin_files = {'text': './tests/data/enwik8_first100_text_document.bin'} [/STATE]
            args.output_prefix, key, "document"
        )
        output_idx_files[key] = "{}_{}_{}.idx".format( # [STATE] output_idx_files = {'text': './tests/data/enwik8_first100_text_document.idx'} [/STATE]
            args.output_prefix, key, "document"
        )
        builders[key] = indexed_dataset.make_builder( # [STATE] builders = {'text': <megatron.data.indexed_dataset.MMapIndexedDatasetBuilder object at 0x7f1a969bfcd0>} [/STATE]
            output_bin_files[key],
            impl=args.dataset_impl,
            vocab_size=tokenizer.vocab_size,
        )

    # actually do tokenization
    proc_start = time.time() # [STATE] proc_start = 1712171758.7156355 [/STATE]
    total_bytes_processed = 0 # [STATE] total_bytes_processed = 0 [/STATE]
    pbar = tqdm.tqdm() # [STATE] pbar = {iterable=None, desc='', total=None, leave=True, fp=<tqdm.utils.DisableOnWriteError object at 0x7f1a969bf100>, ncols=None, nrows=None, mininterval=0.1, maxinterval=10.0, miniters=0, dynamic_miniters=True, ascii=True, disable=False, unit='it', unit_scale=False, unit_divisor=1000, initial=0, lock_args=None, delay=0.0, gui=False, dynamic_ncols=False, smoothing=0.3, _ema_dn=<tqdm.std.EMA object at 0x7f1a969bf520>, _ema_dt=<tqdm.std.EMA object at 0x7f1a969bf3a0>, _ema_miniters=<tqdm.std.EMA object at 0x7f1a969bf070>, bar_format=None, postfix=None, colour=None, _time=<built-in function time>, last_print_n=0, n=0, pos=0, last_print_t=1712171758.7435398, start_t=1712171758.7435398} [/STATE]
    for i, (doc, bytes_processed) in enumerate(encoded_docs, start=1): #  [STATE] 2 [/STATE]
        total_bytes_processed += bytes_processed # [STATE] total_bytes_processed = 3355 [/STATE]

        # release semaphore so `yield_from_files` can add another file to the buffer
        semaphore.release() # [STATE] semaphore = {_cond=<Condition(<unlocked _thread.lock object at 0x7f1a969bf450>, 0)>, _value=10000} [/STATE]

        # add each tokenized document / sentence
        for key, sentences in doc.items(): #  [STATE] 1 [/STATE]
            for sentence in sentences: #  [STATE] 1 [/STATE]
                builders[key].add_item(np.array(sentence, dtype=builders[key].dtype))
            # separate with eos token
            builders[key].end_document()

        # log progress
        if i % args.log_interval == 0:
            current = time.time()
            elapsed = current - proc_start
            mbs = total_bytes_processed / elapsed / 1024 / 1024
            pbar.set_description(
                f"Processed {i}{'' if args.num_docs is None else '/' + str(args.num_docs)} documents ({i / elapsed :.2f} docs/s, {mbs:.2f} MB/s)."
            )
            if i != 0:
                pbar.update(args.log_interval)

    # save output file
    for key in args.jsonl_keys:
        builders[key].finalize(output_idx_files[key])

main(['--input', './tests/data/enwik8_first100.txt', '--output-prefix', './tests/data/enwik8_first100', '--vocab', 'gpt2', '--tokenizer-type', 'HFGPT2Tokenizer', '--merge-file', './data/gpt2-merges.txt', '--append-eod'])

1. def main(input_args=None):
2.     args = get_args(input_args)
3.     encoder = Encoder(args)
4.     tokenizer = build_tokenizer(args)
5.     print(f"Vocab size: {tokenizer.vocab_size}")
6.     print(f"Output prefix: {args.output_prefix}")
7.
8.     # build a semaphore object to stop `yield_from_files` from getting ahead of encoder.encode and
9.     # hence building up memory
10.     semaphore = Semaphore(10000 + args.workers)
11.
12.     # use multiprocessing to iterate over input documents
13.     fin = yield_from_files(args.input.split(","), semaphore)
14.
15.     if args.workers > 1:
16.         pool = multiprocessing.Pool(args.workers, initializer=encoder.initializer)
17.         encoded_docs = pool.imap(encoder.encode, fin, chunksize=25)
18.     else:
19.         encoder.initializer()
20.         encoded_docs = (encoder.encode(doc) for doc in fin)
21.
22.     # make a dataset builder for each key in args.jsonl_keys
23.     # each key will output to a different file beginning with args.output_prefix
24.     output_bin_files = {}
25.     output_idx_files = {}
26.     builders = {}
27.     for key in args.jsonl_keys:
28.         output_bin_files[key] = "{}_{}_{}.bin".format(
29.             args.output_prefix, key, "document"
30.         )
31.         output_idx_files[key] = "{}_{}_{}.idx".format(
32.             args.output_prefix, key, "document"
33.         )
34.         builders[key] = indexed_dataset.make_builder(
35.             output_bin_files[key],
36.             impl=args.dataset_impl,
37.             vocab_size=tokenizer.vocab_size,
38.         )
39.
40.     # actually do tokenization
41.     proc_start = time.time()
42.     total_bytes_processed = 0
43.     pbar = tqdm.tqdm()
44.     for i, (doc, bytes_processed) in enumerate(encoded_docs, start=1):
45.         total_bytes_processed += bytes_processed
46.
47.         # release semaphore so `yield_from_files` can add another file to the buffer
48.         semaphore.release()
49.
50.         # add each tokenized document / sentence
51.         for key, sentences in doc.items():
52.             for sentence in sentences:
53.                 builders[key].add_item(np.array(sentence, dtype=builders[key].dtype))
54.             # separate with eos token
55.             builders[key].end_document()
56.
57.         # log progress
58.         if i % args.log_interval == 0:
59.             current = time.time()
60.             elapsed = current - proc_start
61.             mbs = total_bytes_processed / elapsed / 1024 / 1024
62.             pbar.set_description(
63.                 f"Processed {i}{'' if args.num_docs is None else '/' + str(args.num_docs)} documents ({i / elapsed :.2f} docs/s, {mbs:.2f} MB/s)."
64.             )
65.             if i != 0:
66.                 pbar.update(args.log_interval)
67.
68.     # save output file
69.     for key in args.jsonl_keys:
70.         builders[key].finalize(output_idx_files[key])
71.
72. main(['--input', './tests/data/enwik8_first100.txt', '--output-prefix', './tests/data/enwik8_first100', '--vocab', 'gpt2', '--tokenizer-type', 'HFGPT2Tokenizer', '--merge-file', './data/gpt2-merges.txt', '--append-eod'])

Question: How many times will the loop on line 52 execute when 'main(['--input', './tests/data/enwik8_first100.txt', '--output-prefix', './tests/data/enwik8_first100', '--vocab', 'gpt2', '--tokenizer-type', 'HFGPT2Tokenizer', '--merge-file', './data/gpt2-merges.txt', '--append-eod'])' is called?

Answer: 1