| checkpoints: | |
| checkpoint_interval: 1000 | |
| checkpoints_path: /fsx/phuc/new_workspace/experiments/exp57_8b_llama_1024_ctx_length_and_64_segment_length_and_100k_bs_and_global_lr_1.0e-5_and_balance_factor_lr_0.01_and_balance_factor_0_weight_decay/checkpoints | |
| checkpoints_path_is_shared_file_system: true | |
| resume_checkpoint_path: /fsx/phuc/new_workspace/experiments/infini_attention_8b_llama/exp57_8b_llama_1024_ctx_length_and_64_segment_length_and_100k_bs_and_global_lr_1.0e-5_and_balance_factor_lr_0.01_and_balance_factor_0_weight_decay/checkpoints | |
| save_initial_state: false | |
| data: | |
| dataset: | |
| dataloader_type: single | |
| dataset_max_tokens: null | |
| dataset_weights: | |
| - 0.3 | |
| - 0.3 | |
| - 0.45 | |
| - 0.15 | |
| - 0.08 | |
| - 0.02 | |
| datasets: | |
| - dtype: uint32 | |
| filename_pattern: .*.ds | |
| folder: s3://huggingface-llm-datasets/stack_full_v21-8k/tokenized-llama3/long/ | |
| skip_tokens: 0 | |
| - dtype: uint32 | |
| filename_pattern: .*.ds | |
| folder: s3://huggingface-llm-datasets/stack_full_v21-8k/tokenized-llama3/short/ | |
| skip_tokens: 0 | |
| - dtype: uint32 | |
| filename_pattern: .*.ds | |
| folder: s3://huggingface-llm-datasets/fineweb-v1-8k/tokenized-llama3/long/CC-MAIN-2024-10 | |
| skip_tokens: 0 | |
| - dtype: uint32 | |
| filename_pattern: .*.ds | |
| folder: s3://huggingface-llm-datasets/fineweb-v1-8k/tokenized-llama3/short/CC-MAIN-2024-10 | |
| skip_tokens: 0 | |
| - dtype: uint32 | |
| filename_pattern: .*.ds | |
| folder: s3://huggingface-llm-datasets/project-gutenberg/tokenized-llama3/ | |
| skip_tokens: 0 | |
| - dtype: uint32 | |
| filename_pattern: .*.ds | |
| folder: s3://huggingface-llm-datasets/OpenHermes-2-5/tokenized-llama3 | |
| skip_tokens: 0 | |
| pad_samples_to_global_batch_size: false | |
| skip_in_stream: true | |
| num_loading_workers: 0 | |
| seed: 42 | |
| data_stages: null | |
| experiment_logger: | |
| tensorboard_logger: | |
| flush_secs: 30 | |
| tensorboard_dir: /fsx/phuc/project_data/infini_attention/tb_logs | |
| wandb_logger: | |
| wandb_entity: null | |
| wandb_project: infini_attention_8b_llama | |
| general: | |
| benchmark_csv_path: null | |
| consumed_train_samples: 1920000 | |
| ignore_sanity_checks: true | |
| project: infini_attention_8b_llama | |
| run: exp57_8b_llama_1024_ctx_length_and_64_segment_length_and_100k_bs_and_global_lr_1.0e-5_and_balance_factor_lr_0.01_and_balance_factor_0_weight_decay | |
| seed: 42 | |
| step: 20000 | |
| infini_attention: | |
| balance_act_type: orig_sigmoid | |
| balance_factor_lr: 0.01 | |
| balance_factor_weight_decay: 0.0 | |
| balance_init_type: zeros | |
| log_grad: false | |
| log_segment_acts: false | |
| logging: true | |
| logging_interval: 250 | |
| segment_length: 64 | |
| turn_on_memory: true | |
| kill_switch_path: null | |
| lighteval: null | |
| logging: | |
| iteration_step_info_interval: 1 | |
| log_level: info | |
| log_level_replica: info | |
| model: | |
| ddp_bucket_cap_mb: 25 | |
| dtype: bfloat16 | |
| init_method: | |
| path: /fsx/phuc/projects/infini-attention/llama3-ckps/haojun-8b-llama-nanotron-ckp/NanotronLlama3-8B | |
| make_vocab_size_divisible_by: 1 | |
| model_config: | |
| bos_token_id: 128000 | |
| eos_token_id: 128001 | |
| hidden_act: silu | |
| hidden_size: 4096 | |
| initializer_range: 0.02 | |
| intermediate_size: 14336 | |
| is_llama_config: true | |
| max_position_embeddings: 8192 | |
| num_attention_heads: 32 | |
| num_hidden_layers: 32 | |
| num_key_value_heads: 8 | |
| pad_token_id: null | |
| pretraining_tp: 1 | |
| rms_norm_eps: 1.0e-05 | |
| rope_interleaved: false | |
| rope_scaling: null | |
| rope_theta: 500000.0 | |
| tie_word_embeddings: false | |
| use_cache: true | |
| vocab_size: 128256 | |
| optimizer: | |
| accumulate_grad_in_fp32: false | |
| adam_beta1: 0.9 | |
| adam_beta2: 0.95 | |
| adam_eps: 1.0e-08 | |
| clip_grad: 1.0 | |
| learning_rate_scheduler: | |
| learning_rate: 1.0e-05 | |
| lr_decay_starting_step: null | |
| lr_decay_steps: 23500 | |
| lr_decay_style: cosine | |
| lr_warmup_steps: 1500 | |
| lr_warmup_style: linear | |
| min_decay_lr: 1.0e-06 | |
| torch_adam_is_fused: true | |
| weight_decay: 0.1 | |
| zero_stage: 0 | |
| parallelism: | |
| dp: 6 | |
| expert_parallel_size: 1 | |
| pp: 1 | |
| pp_engine: 1f1b | |
| tp: 4 | |
| tp_linear_async_communication: false | |
| tp_mode: ALL_REDUCE | |
| profiler: null | |
| s3_upload: | |
| remove_after_upload: true | |
| s5cmd_concurrency: 5 | |
| s5cmd_numworkers: 16 | |
| s5cmd_path: null | |
| upload_s3_path: s3://phuc-experiments/infini-attention/8b-llama/exp57_8b_llama_1024_ctx_length_and_64_segment_length_and_100k_bs_and_global_lr_1.0e-5_and_balance_factor_lr_0.01_and_balance_factor_0_weight_decay | |
| tokenizer: | |
| tokenizer_max_length: null | |
| tokenizer_name_or_path: /fsx/haojun/lighteval_evaluation_model/NanotronLlama3-8B | |
| tokenizer_revision: null | |
| tokens: | |
| batch_accumulation_per_replica: 1 | |
| limit_test_batches: 0 | |
| limit_val_batches: 0 | |
| micro_batch_size: 16 | |
| sequence_length: 1024 | |
| train_steps: 25000 | |
| val_check_interval: -1 | |