Project Name: EleutherAI+gpt-neox
Language: Python
Statement Type: Branch

Source Code:
def test_model_training_options(monkeypatch, key, value):
    # TODO: Possibly add testing over world_size=2 back in
    neox_args = NeoXArgs.from_dict(BASE_CONFIG)
    if getattr(neox_args, key) == value:
        pytest.skip("Skipping to avoid redundancy as no change in base config")
    if key == "precision" and value == "bfloat16":
        pytest.xfail(
            reason="Assumes that ZeRO optimization stage has been set in the YAML"
        )
    param_dict = {key: value}
    run_train_test(monkeypatch, overwrite_values=param_dict)

test_model_training_options(monkeypatch={_setattr=[], _setitem=[], _cwd=None, _savesyspath=None}, key='gpt_j_residual', value=False)

Selected Statement: if getattr(neox_args, key) == value:

Function Input:
monkeypatch: {_setattr=[], _setitem=[], _cwd=None, _savesyspath=None}
key: 'gpt_j_residual'
value: False

Variable Values Before Statement:
value: False

Value After Statement Execution: Yes

Variable States During Runtime (line, value):
monkeypatch: 1
{_setattr=[], _setitem=[], _cwd=None, _savesyspath=None}
key: 1
'gpt_j_residual'
value: 1
False
neox_args: 3
NeoXArgs(distributed_backend='nccl', local_rank=None, rank=None, lazy_mpu_init=False, short_seq_prob=0.1, eod_mask_loss=False, adlr_autoresume=False, adlr_autoresume_interval=1000, seed=1234, onnx_safe=False, deepscale=False, deepscale_config=None, deepspeed_mpi=False, deepspeed_slurm=False, user_script=None, iteration=None, do_train=None, do_valid=None, do_test=None, save_iters=[], global_num_gpus=1, text_gen_type='unconditional', temperature=0.0, top_p=0.0, top_k=0, return_logits=False, maximum_tokens=64, prompt_end='\n', sample_input_file=None, sample_output_file='samples.txt', num_samples=1, recompute=False, eval_results_prefix='', eval_tasks=None, use_wandb=False, wandb_group=None, wandb_team=None, wandb_project='neox', wandb_host='https://api.wandb.ai', wandb_init_all_ranks=False, git_hash='7a8fa2f0', log_dir=None, tensorboard_dir=None, log_interval=10, log_grad_pct_zeros=False, log_param_norm=False, log_grad_norm=False, log_optimizer_states=False, log_gradient_noise_scale=False, gradient_noise_scale_n_batches=5, gradient_noise_scale_cpu_offload=False, pipe_parallel_size=1, model_parallel_size=1, pipe_partition_method='type:transformer|mlp', world_size=None, is_pipe_parallel=True, data_path='data/enwik8/enwik8_text_document', use_shared_fs=True, train_data_paths=None, label_data_paths=None, test_data_paths=None, valid_data_paths=None, train_data_weights=None, valid_data_weights=None, test_data_weights=None, weight_by_num_documents=False, weighted_sampler_alpha=1.0, data_impl='mmap', mmap_warmup=False, save=None, s3_path=None, s3_chunk_size=104857600, config_files=None, load=None, checkpoint_validation_with_forward_pass=False, checkpoint_scale='linear', checkpoint_factor=1000, extra_save_iters=None, no_save_optim=False, no_save_rng=False, no_load_optim=False, no_load_rng=False, finetune=False, batch_size=4, train_iters=20, eval_iters=10, keep_last_n_checkpoints=None, eval_interval=100000, split='969, 30, 1', vocab_file='data/gpt2-vocab.json', merge_file='data/gpt2-merges.txt', num_workers=1, exit_interval=None, attention_dropout=0, hidden_dropout=0, weight_decay=0.1, checkpoint_activations=True, checkpoint_num_layers=1, deepspeed_activation_checkpointing=True, contiguous_checkpointing=False, checkpoint_in_cpu=False, synchronize_each_layer=True, profile_backward=False, partition_activations=True, gas=1, clip_grad=1.0, hysteresis=2, dynamic_loss_scale=True, loss_scale=None, loss_scale_window=1000.0, min_scale=1.0, char_level_ppl=False, use_mup=False, coord_check=False, save_base_shapes=False, base_shapes_file=None, mup_init_scale=1.0, mup_attn_temp=1.0, mup_output_temp=1.0, mup_embedding_mult=1.0, mup_rp_embedding_mult=1.0, mup_width_scale=2, tokenizer_type='GPT2BPETokenizer', padded_vocab_size=None, optimizer_type='sm3', use_bnb_optimizer=False, zero_stage=0, zero_reduce_scatter=True, zero_contiguous_gradients=False, zero_reduce_bucket_size=500000000, zero_allgather_bucket_size=500000000, lr=0.001, lr_decay_style='cosine', lr_decay_iters=20, min_lr=0.0, warmup=0.01, override_lr_scheduler=False, use_checkpoint_lr_scheduler=False, precision='fp16', num_layers=2, hidden_size=8, num_attention_heads=4, seq_length=1024, max_position_embeddings=1024, norm='layernorm', use_qk_layernorm=False, layernorm_epsilon=1e-05, rms_norm_epsilon=1e-08, scalenorm_epsilon=1e-08, pos_emb='rotary', rpe_num_buckets=32, rpe_max_distance=128, opt_pos_emb_offset=0, no_weight_tying=True, attention_config=['global', 'global'], sparsity_config={}, num_unique_layers=None, param_sharing_style='grouped', make_vocab_size_divisible_by=128, activation='gelu', scaled_upper_triang_masked_softmax_fusion=False, scaled_masked_softmax_fusion=False, bias_gelu_fusion=False, bias_dropout_fusion=False, rope_fusion=False, fp16_lm_cross_entropy=False, init_method_std=0.02, apply_query_key_layer_scaling=False, use_cpu_initialization=False, attention_softmax_in_fp32=False, rotary_pct=1.0, rotary_emb_base=10000, init_method='small_init', output_layer_init_method='wang_init', gmlp_attn_dim=64, gpt_j_residual=False, gpt_j_tied=False, use_bias_in_norms=True, use_bias_in_attn_linear=True, mlp_type='regular', soft_prompt_tuning=None, output_layer_parallelism='column', deepspeed=True, train_batch_size=4, train_micro_batch_size_per_gpu=4, gradient_accumulation_steps=1, optimizer={'type': 'sm3', 'params': {}}, scheduler=None, fp32_allreduce=False, prescale_gradients=False, gradient_predivide_factor=1.0, sparse_gradients=False, fp16={'type': 'fp16', 'enabled': True}, bf16=None, amp=None, gradient_clipping=1.0, zero_optimization={'stage': 0, 'allgather_partitions': True, 'reduce_scatter': True, 'allgather_bucket_size': 500000000, 'overlap_comm': False, 'reduce_bucket_size': 500000000, 'contiguous_gradients': False}, curriculum_learning=None, curriculum_seqlen=0, steps_per_print=10, wall_clock_breakdown=True, dump_state=False, flops_profiler=None, communication_data_type=None, autotuning=None, activation_checkpointing=None, sparse_attention=None, data_efficiency=None, tensorboard=None, wandb=None, csv_monitor=None, elasticity=None, comms_logger=None, compression_training=None, checkpoint=None, data_types=None, deepspeed_extra_args={'comms_logger': {'enabled': True, 'verbose': True, 'prof_all': True, 'debug': False}}, hostfile='None', include='localhost:1', exclude=None, num_nodes=-1, num_gpus=None, master_port=29500, master_addr=None, launcher='pdsh', force_multi=False, detect_nvlink_pairs=False, autotuning_run=None, no_ssh_check=False, comment=None, account=None)