model_config.json

{
  "hiddenSize": 640,
  "numLayers": 8,
  "numAttentionHeads": 14,
  "numKeyValueHeads": 2,
  "numExperts": 128,
  "topK": 4,
  "vocabSize": 200064,
  "maxPositionEmbeddings": 131072,
  "numLabels": 33
}