checkpoints:
  checkpoint_interval: 1000
  checkpoints_path: /fsx/phuc/new_workspace/experiments/exp57_8b_llama_1024_ctx_length_and_64_segment_length_and_100k_bs_and_global_lr_1.0e-5_and_balance_factor_lr_0.01_and_balance_factor_0_weight_decay/checkpoints
  checkpoints_path_is_shared_file_system: true
  resume_checkpoint_path: /fsx/phuc/new_workspace/experiments/infini_attention_8b_llama/exp57_8b_llama_1024_ctx_length_and_64_segment_length_and_100k_bs_and_global_lr_1.0e-5_and_balance_factor_lr_0.01_and_balance_factor_0_weight_decay/checkpoints
  save_initial_state: false
data:
  dataset:
    dataloader_type: single
    dataset_max_tokens: null
    dataset_weights:
    - 0.3
    - 0.3
    - 0.45
    - 0.15
    - 0.08
    - 0.02
    datasets:
    - dtype: uint32
      filename_pattern: .*.ds
      folder: s3://huggingface-llm-datasets/stack_full_v21-8k/tokenized-llama3/long/
      skip_tokens: 0
    - dtype: uint32
      filename_pattern: .*.ds
      folder: s3://huggingface-llm-datasets/stack_full_v21-8k/tokenized-llama3/short/
      skip_tokens: 0
    - dtype: uint32
      filename_pattern: .*.ds
      folder: s3://huggingface-llm-datasets/fineweb-v1-8k/tokenized-llama3/long/CC-MAIN-2024-10
      skip_tokens: 0
    - dtype: uint32
      filename_pattern: .*.ds
      folder: s3://huggingface-llm-datasets/fineweb-v1-8k/tokenized-llama3/short/CC-MAIN-2024-10
      skip_tokens: 0
    - dtype: uint32
      filename_pattern: .*.ds
      folder: s3://huggingface-llm-datasets/project-gutenberg/tokenized-llama3/
      skip_tokens: 0
    - dtype: uint32
      filename_pattern: .*.ds
      folder: s3://huggingface-llm-datasets/OpenHermes-2-5/tokenized-llama3
      skip_tokens: 0
    pad_samples_to_global_batch_size: false
    skip_in_stream: true
  num_loading_workers: 0
  seed: 42
data_stages: null
experiment_logger:
  tensorboard_logger:
    flush_secs: 30
    tensorboard_dir: /fsx/phuc/project_data/infini_attention/tb_logs
  wandb_logger:
    wandb_entity: null
    wandb_project: infini_attention_8b_llama
general:
  benchmark_csv_path: null
  consumed_train_samples: 1920000
  ignore_sanity_checks: true
  project: infini_attention_8b_llama
  run: exp57_8b_llama_1024_ctx_length_and_64_segment_length_and_100k_bs_and_global_lr_1.0e-5_and_balance_factor_lr_0.01_and_balance_factor_0_weight_decay
  seed: 42
  step: 20000
infini_attention:
  balance_act_type: orig_sigmoid
  balance_factor_lr: 0.01
  balance_factor_weight_decay: 0.0
  balance_init_type: zeros
  log_grad: false
  log_segment_acts: false
  logging: true
  logging_interval: 250
  segment_length: 64
  turn_on_memory: true
kill_switch_path: null
lighteval: null
logging:
  iteration_step_info_interval: 1
  log_level: info
  log_level_replica: info
model:
  ddp_bucket_cap_mb: 25
  dtype: bfloat16
  init_method:
    path: /fsx/phuc/projects/infini-attention/llama3-ckps/haojun-8b-llama-nanotron-ckp/NanotronLlama3-8B
  make_vocab_size_divisible_by: 1
  model_config:
    bos_token_id: 128000
    eos_token_id: 128001
    hidden_act: silu
    hidden_size: 4096
    initializer_range: 0.02
    intermediate_size: 14336
    is_llama_config: true
    max_position_embeddings: 8192
    num_attention_heads: 32
    num_hidden_layers: 32
    num_key_value_heads: 8
    pad_token_id: null
    pretraining_tp: 1
    rms_norm_eps: 1.0e-05
    rope_interleaved: false
    rope_scaling: null
    rope_theta: 500000.0
    tie_word_embeddings: false
    use_cache: true
    vocab_size: 128256
optimizer:
  accumulate_grad_in_fp32: false
  adam_beta1: 0.9
  adam_beta2: 0.95
  adam_eps: 1.0e-08
  clip_grad: 1.0
  learning_rate_scheduler:
    learning_rate: 1.0e-05
    lr_decay_starting_step: null
    lr_decay_steps: 23500
    lr_decay_style: cosine
    lr_warmup_steps: 1500
    lr_warmup_style: linear
    min_decay_lr: 1.0e-06
  torch_adam_is_fused: true
  weight_decay: 0.1
  zero_stage: 0
parallelism:
  dp: 6
  expert_parallel_size: 1
  pp: 1
  pp_engine: 1f1b
  tp: 4
  tp_linear_async_communication: false
  tp_mode: ALL_REDUCE
profiler: null
s3_upload:
  remove_after_upload: true
  s5cmd_concurrency: 5
  s5cmd_numworkers: 16
  s5cmd_path: null
  upload_s3_path: s3://phuc-experiments/infini-attention/8b-llama/exp57_8b_llama_1024_ctx_length_and_64_segment_length_and_100k_bs_and_global_lr_1.0e-5_and_balance_factor_lr_0.01_and_balance_factor_0_weight_decay
tokenizer:
  tokenizer_max_length: null
  tokenizer_name_or_path: /fsx/haojun/lighteval_evaluation_model/NanotronLlama3-8B
  tokenizer_revision: null
tokens:
  batch_accumulation_per_replica: 1
  limit_test_batches: 0
  limit_val_batches: 0
  micro_batch_size: 16
  sequence_length: 1024
  train_steps: 25000
  val_check_interval: -1