skimo2022/anlp-a2-p1-moe_4e_top1

Advanced NLP Assignment 2, Part 1: moe_4e_top1.

Decoder-only Transformer translating Vietnamese / Japanese -> English, trained from scratch on belumind/en-vi-ja-curated-500k-triplets.

Prompt format: <bos> <vi|ja> {source} <2en> -> model generates English until <eos>.

eval_results.json

{
 "variant": "moe_4e_top1",
 "params": {
  "total": 35277312,
  "active": 25840128,
  "non_embedding_total": 18893312,
  "non_embedding_active": 9456128
 },
 "test_ppl": {
  "vi": 4.377480342983202,
  "ja": 6.058690242385236,
  "all": 5.149931789865328
 },
 "bleu_vi": 41.58798840044216,
 "chrf_vi": 61.49076943115253,
 "bleu_ja": 30.915491370941474,
 "chrf_ja": 54.61319910787728,
 "bleu_all": 36.23031219970891,
 "n_bleu_examples": 10000,
 "decode_time_s": 138.4640130996704,
 "train": {
  "tokens_trained": 110009907,
  "steps": 10456,
  "epochs": 2.999093074141973,
  "train_time_s": 2960.1608505249023,
  "tokens_per_s": 37163.489605807335,
  "params": {
   "total": 35277312,
   "active": 25840128,
   "non_embedding_total": 18893312,
   "non_embedding_active": 9456128
  }
 },
 "load_cv": 0.10999769885240329,
 "lang_expert_mutual_info_bits": [
  0.024267374606437894,
  0.0015941166241417015,
  0.11267233065255515,
  0.020869713206173254,
  0.07364382731686486,
  0.003974228269156561
 ]
}

train_summary.json

{
 "tokens_trained": 110009907,
 "steps": 10456,
 "epochs": 2.999093074141973,
 "train_time_s": 2960.1608505249023,
 "tokens_per_s": 37163.489605807335,
 "params": {
  "total": 35277312,
  "active": 25840128,
  "non_embedding_total": 18893312,
  "non_embedding_active": 9456128
 }
}

Load with Transformer.load(<local snapshot dir>) from src/common/model.py of the course repo.