{
    "async_checkpointing": false,
    "async_eval_ngpus": -1,
    "batch_size": 4,
    "data": "",
    "disable_logging": false,
    "disable_workers_print": false,
    "dtype": "bf16",
    "dump_after_steps": 0,
    "dump_dir": "/fsx-onellm/rpasunuru/SFT/v2.1_textpp_30b_730k_sftv1.4_exp1/v2.1_textpp_30b_730k_sftv1.4_exp1_run000",
    "dump_freq": 400,
    "dump_profile_traces": false,
    "enable_loss_tracker": false,
    "epochs": -1,
    "eval_freq": 100000,
    "exp_id": "",
    "exp_name": "",
    "finetuning_dir": "/fsx-onellm/shared/from_rsc//v2.1_30b_qk_zloss_snorm_Nov_26_3_run000_checkpoint_0730000",
    "fp32_reduce_scatter": "all",
    "gpu_check_level": 3,
    "image_loss_weight": 1.0,
    "image_text_rotation_prob": 0.0,
    "instruct": {
        "no_loss_prompt": true,
        "no_loss_truncated": false,
        "use_eot": true
    },
    "instruct_data": "/fsx-onellm/shared/finetuning/sft_v1.4_data/splits/train/long_caption:2.92,/fsx-onellm/shared/finetuning/sft_v1.4_data/splits/train/vqa:4.59,/fsx-onellm/shared/finetuning/sft_v1.4_data/splits/train/text2image:10.44,/fsx-onellm/shared/finetuning/sft_v1.4_data/splits/train/llama2_rjv6_helpful:43.27,/fsx-onellm/shared/finetuning/sft_v1.4_data/splits/train/code_llama:0.51,/fsx-onellm/shared/finetuning/sft_v1.4_data/splits/train/interleaved_batch1-17:27.45,/fsx-onellm/shared/finetuning/sft_v1.4_data/splits/train/image_dialogue:7.46,/fsx-onellm/shared/finetuning/sft_v1.4_data/splits/train/llama2_rjv6_harmless:0.97,/fsx-onellm/shared/finetuning/sft_v1.4_data/splits/train/cybersec_safety:0.33,/fsx-onellm/shared/finetuning/sft_v1.4_data/splits/train/onellm_multimodal_safety:0.86,/fsx-onellm/shared/finetuning/sft_v1.4_data/splits/train/autosafety:0.51,/fsx-onellm/shared/finetuning/sft_v1.4_data/splits/train/rainbow_safety:0.10,/fsx-onellm/shared/finetuning/sft_v1.4_data/splits/train/genai_safety:0.58",
    "iter_gopher": {
        "buffer_size": 16,
        "max_precompute": 10,
        "n_chars_by_tok": 15,
        "n_seqs_to_concat": 10,
        "num_processes": 1
    },
    "iter_jsonl": {
        "buffer_size": 64,
        "same_data": false
    },
    "iter_multi": {
        "buffer_size": 512,
        "ignore_extra_chunks": true,
        "max_precompute": 20,
        "multiprocess": true
    },
    "iter_type": "multi",
    "keep_checkpoints_every_steps": 400,
    "keep_eval_checkpoints": true,
    "keep_n_last_checkpoints": 2,
    "log_all_steps": false,
    "log_freq": 10,
    "log_updates": true,
    "log_wandb": false,
    "loss_rescaling": false,
    "model": {
        "add_extra_toks": "0",
        "alpha_depth": "disabled",
        "attn_dropout": 0,
        "attn_to_keep": "all",
        "custom_bwd": false,
        "dim": 8192,
        "dropout": 0.05,
        "efficient_attn": "flash",
        "emb_dropout": 0,
        "ffn_dim_multiplier": 1.0,
        "ffn_dropout": 0,
        "full_logging_n_layers": 4,
        "fuse_sequence_parallel": false,
        "init": {
            "coeff_std": null,
            "depth_last": false,
            "fixed_std": null,
            "no_init": false,
            "pos_init_scalar": null,
            "use_depth": "current",
            "use_gaussian": true
        },
        "layer_ckpt": "0::2",
        "linear_residual_dropout": false,
        "loss_parallel": true,
        "max_length": 2048,
        "multiple_of": 256,
        "n_heads": 64,
        "n_kv_heads": 8,
        "n_layers": 48,
        "non_linearity": "swiglu",
        "norm_affine": true,
        "norm_eps": 1e-05,
        "norm_type": "rmsnorm",
        "output_dropout": 0,
        "output_size": -1,
        "pre_norm": true,
        "qk_normalization": true,
        "recompute_attn": true,
        "recompute_fc1_out": true,
        "recompute_fc3_out": true,
        "residual_dropout": 0.0,
        "rope_theta": 10000.0,
        "sequence_parallel": false,
        "swin_norm": true,
        "turn_eos_token": "<eos>",
        "use_rope": true,
        "vocab_size": 65536
    },
    "model_parallel_size": 4,
    "no_final_ckpt": false,
    "num_retrieved_docs": 0,
    "old_mp": -1,
    "old_world_size": -1,
    "optim": {
        "beta1": 0.9,
        "beta2": 0.95,
        "clip": 1.0,
        "cosine_theta": 1.0,
        "cycle_length": 1.0,
        "epsilon": 1e-08,
        "exp_factor": 0.5,
        "lr": 1e-05,
        "lr_min_ratio": 0.1,
        "scheduler": "cosine",
        "use_deprecated_optim": false,
        "warmup": 100,
        "weight_decay": 0.1
    },
    "periodic_gpu_check": true,
    "profile_freq": -1,
    "reshard_after_forward": true,
    "restore_dataloader_position": false,
    "retrieval_prob": 0.0,
    "rlhf": null,
    "root_dump_dir": "",
    "save_optimizer_states": true,
    "seq_len": 4096,
    "slurm": {
        "global_rank": 0,
        "is_slurm_job": true,
        "world_size": 128
    },
    "steps": 1200,
    "tokenizer": "/fsx-onellm/rpasunuru/models/cm3z/cm3v2_7b_placeholder/gpt2-unified-image-sentinel.json",
    "tokenizer_dir": "/fsx/guismay/data/large_experiments/fair_llm/datasets/tokenizers",
    "torch_seed": -1,
    "unlimited_steps": false,
    "use_hf_tokenizer": true,
    "valid": {
        "batch_size": 1,
        "debug": false,
        "majority_voting": 0,
        "n_batches": 100,
        "onellm_eval": false,
        "onellm_eval_media_storage": "",
        "ppl_files_str": "",
        "prompt_path": "",
        "prompt_templates": "{}",
        "random_fewshots": false,
        "seq_len": 4096,
        "tasks_root_dir": "",
        "tasks_str": "",
        "temperature": 1.0,
        "top_k": 0,
        "top_p": 0.0,
        "use_sampling": false,
        "write_eval": false
    },
    "wandb_entity": "violet-zct",
    "wandb_project": "instruct_sft",
    "water_marking_codes_str": null,
    "z_loss_weight": 0.0001
}