t5x/examples/decoder_only/models/xxl.gin

# Decoder-only model (XXL) with 4762357760 parameters.

include 't5x/examples/decoder_only/models/base.gin'  # imports vocab, optimizer and model.

# ------------------- Network specification overrides --------------------------
network.TransformerConfig:
  emb_dim = 4096
  num_heads = 64
  num_layers = 24
  head_dim = 64
  mlp_dim = 10240