dnebh/anlp-a2-part2-mars

ANLP A2 Part 2 — mars

Dense decoder-only Transformer (Part 1 config 1, 33.4M params) trained for next-token prediction on browndw/human-ai-parallel-corpus with a from-scratch mars optimizer (lr 0.001).

Final: val loss 3.1919, test BLEU 0.8750586542990738, tokens 48,496,640.

Load with src.part1.hub.load_exported_model(<folder>) from the assignment repo.

{
  "optimizer": "mars",
  "lr": 0.001,
  "hparams": {
    "betas": [
      0.9,
      0.95
    ],
    "eps": 1e-08,
    "weight_decay": 0.1,
    "gamma": 0.025
  },
  "fraction": 1.0,
  "batch_size": 64,
  "total_steps": 2960,
  "warmup_steps": 296,
  "tokens_per_step": 16384,
  "dataset_tokens": 48512768,
  "n_params": 33358848,
  "seed": 42,
  "data_stats": {
    "n_rows": 66320,
    "n_base_docs": {
      "test": 414,
      "val": 414,
      "train": 7462
    },
    "train_stream_tokens": 48512889,
    "val_stream_tokens": 2698138,
    "n_train_windows": 189503,
    "n_val_windows": 10539,
    "n_bleu_items": 414,
    "seq_len": 256,
    "seed": 0
  },
  "curve": [
    {
      "step": 0,
      "tokens": 0,
      "dataset_fraction": 0.0,
      "val_loss": 9.806513278757398,
      "val_ppl": 18151.587088404638,
      "val_tokens": 2697984,
      "test_bleu": 0.14436034784732546,
      "eval_time_s": 32.27794098854065
    },
    {
      "step": 296,
      "tokens": 4849664,
      "dataset_fraction": 0.09996675514371804,
      "val_loss": 4.684496766555602,
      "val_ppl": 108.25578066894413,
      "val_tokens": 2697984,
      "test_bleu": 0.3750821728312126,
      "eval_time_s": 31.82180142402649
    },
    {
      "step": 592,
      "tokens": 9699328,
      "dataset_fraction": 0.19993351028743608,
      "val_loss": 4.034365758145712,
      "val_ppl": 56.507069721894936,
      "val_tokens": 2697984,
      "test_bleu": 0.5544116865862202,
      "eval_time_s": 31.830755949020386
    },
    {
      "step": 888,
      "tokens": 14548992,
      "dataset_fraction": 0.2999002654311541,
      "val_loss": 3.7525948049312006,
      "val_ppl": 42.63155918470125,
      "val_tokens": 2697984,
      "test_bleu": 0.43642902779150705,
      "eval_time_s": 31.837430000305176
    },
    {
      "step": 1184,
      "tokens": 19398656,
      "dataset_fraction": 0.39986702057487217,
      "val_loss": 3.5721479517785317,
      "val_ppl": 35.592963071145824,
      "val_tokens": 2697984,
      "test_bleu": 0.6172541428933996,
      "eval_time_s": 31.840679168701172
    },
    {
      "step": 1480,
      "tokens": 24248320,
      "dataset_fraction": 0.4998337757185902,
      "val_loss": 3.4494480386790656,
      "val_ppl": 31.48301010819174,
      "val_tokens": 2697984,
      "test_bleu": 0.6938417283999999,
      "eval_time_s": 31.8419189453125
    },
    {
      "step": 1776,
      "tokens": 29097984,
      "dataset_fraction": 0.5998005308623082,
      "val_loss": 3.352817955636042,
      "val_ppl": 28.583166357365137,
      "val_tokens": 2697984,
      "test_bleu": 0.735729305756493,
      "eval_time_s": 31.837839365005493
    },
    {
      "step": 2072,
      "tokens": 33947648,
      "dataset_fraction": 0.6997672860060263,
      "val_loss": 3.2787492945542858,
      "val_ppl": 26.54255501333499,
      "val_tokens": 2697984,
      "test_bleu": 0.5924428626662617,
      "eval_time_s": 31.84942126274109
    },
    {
      "step": 2368,
      "tokens": 38797312,
      "dataset_fraction": 0.7997340411497443,
      "val_loss": 3.2263052984075333,
      "val_ppl": 25.186428513813528,
      "val_tokens": 2697984,
      "test_bleu": 0.5983417574986116,
      "eval_time_s": 31.841471672058105
    },
    {
      "step": 2664,
      "tokens": 43646976,
      "dataset_fraction": 0.8997007962934623,
      "val_loss": 3.1980376503750114,
      "val_ppl": 24.484435999925424,
      "val_tokens": 2697984,
      "test_bleu": 0.8080309986365949,
      "eval_time_s": 31.84360098838806
    },
    {
      "step": 2960,
      "tokens": 48496640,
      "dataset_fraction": 0.9996675514371804,
      "val_loss": 3.191900298721888,
      "val_ppl": 24.334626592911402,
      "val_tokens": 2697984,
      "test_bleu": 0.8750586542990738,
      "eval_time_s": 31.85160231590271
    }
  ],
  "diverged": false,
  "wandb_url": "https://wandb.ai/dnebhrajani-v/anlp-a2-part2/runs/3zlifir3",
  "hf_url": null,
  "train_time_s": 875.580427646637,
  "final_val_loss": 3.191900298721888
}