kdyck_1pct_50B_d24_seed0_lr0p0036813_2026-07-27_22-29-26_691029-ppt

Trained with nanochat. Checkpoint at step 476.

W&B run: https://wandb.ai/alexksternteam/longhorizon_50B_v0/runs/0dghdwoj

Outcome

metric	value
step	476
smooth_train_loss	3.9976258277893066
min_objective	5.792123248003767
flops_used	2.563585027438805e+18
flops_per_token	5136187392.0
total_training_time	724.637948513031

Training config

{
  "model_pt": {
    "sequence_len": 2048,
    "vocab_size": 65536,
    "n_layer": 24,
    "n_head": 12,
    "n_kv_head": 12,
    "n_embd": 1536
  },
  "model_ppt": {
    "sequence_len": 2048,
    "vocab_size": 256,
    "n_layer": 24,
    "n_head": 12,
    "n_kv_head": 12,
    "n_embd": 1536
  },
  "optim": {
    "matrix_lr": 0.01,
    "embedding_lr": 0.01,
    "unembedding_lr": 0.01,
    "weight_decay": 0.0
  },
  "train": {
    "num_iterations": 1000,
    "eval_every_n_steps": 238,
    "eval_every_flops_frac": null,
    "save_every_n_steps": null,
    "save_at_steps": [
      17166,
      19073,
      36240,
      38147,
      55313,
      57220,
      74387,
      76294,
      95367
    ],
    "log_every_n_steps": 100,
    "eval_at_end": true,
    "save_at_end": true,
    "grad_clip": 1.0,
    "ema_beta": 0.0
  },
  "eval": {
    "eval_steps": 8,
    "eval_tokens": 2016000
  },
  "hardware": {
    "device_batch_size": 32,
    "grad_accum_steps": 4,
    "peak_tflops": 2250.0
  },
  "wandb": {
    "enabled": true,
    "notes": "",
    "group": "5e10tok_ppt0.01",
    "tags": [
      "nanochat_gpt",
      "pt_fineweb-nanochatbpe-55B",
      "ppt_dyck-k128-seq_len_2048-1B",
      "seed_0",
      "case_c",
      "lr_trapezoid",
      "depth_24"
    ],
    "entity": null
  },
  "data_pt": "fineweb-nanochatbpe-55B",
  "data_ppt": "dyck-k128-seq_len_2048-1B",
  "extra_eval": [
    "c4-nanochatbpe-10B"
  ],
  "train_split": "train",
  "pad_vocab": 65536,
  "ppt_vocab_size": null,
  "ppt_same_vocab_as_pt": false,
  "ppt_unified_lr": false,
  "lr_kind": "trapezoid",
  "lr_warmup_ratio": 0.1,
  "lr_warmdown_ratio": 0.4,
  "lr_final_frac": 0.0,
  "ppt_lr_kind": "trapezoid",
  "ppt_lr_warmup_ratio": 0.1,
  "ppt_lr_warmdown_ratio": 0.4,
  "ppt_lr_final_frac": 0.0,
  "ppt_lr": 0.01,
  "ppt_weight_decay": 0.0,
  "ppt_device_batch_size": 32,
  "ppt_grad_accum_steps": 4,
  "eval_tokens": 2016000,
  "reinit_embed_at_transition": true,
  "moment_match_embed_reinit": false,
  "reset_optimizer_at_transition": true,
  "random_pt_head": false,
  "depth": 24,
  "compile_model": true,
  "model_project": "longhorizon_50B_v0",
  "run_name": "kdyck_1pct_50B_d24_seed0_lr0p0036813",
  "target_flops": 0.0,
  "ppt_tokens": 500000000,
  "pt_tokens": 49500000000,
  "ppt_eval_every_n_steps": 238,
  "alpha_ppt": 0.0,
  "use_measured_flops": true,
  "flops_per_token": null,
  "ppt_flops_per_token": null,
  "seed": 0,
  "metrics_output_file": null,
  "push_to_hf": true,
  "hf_repo_org": "alexkstern",
  "cleanup_checkpoints": false
}

Files

• model_000476.pt — model weights (state_dict).
• meta_000476.json — training metadata.
• config_000476.json — run config snapshot.
• rng_000476.pt — RNG state (when present).