{
  "architectures": [
    "FLASHTransformerForPretrained"
  ],
  "auto_map": {
  "AutoConfig": "dnaflash.FLASHTransformerConfig",
  "AutoModel": "dnaflash.FLASHTransformerForPretrained",
  "AutoModelForSequenceClassification": "dnaflash.FLASHTransformerForSequenceClassification"
  },
  "attn_dropout": 0.0,
  "causal": false,
  "expansion_factor": 2.0,
  "group_size": 256,
  "hidden_size": 1024,
  "laplace_attn_fn": false,
  "model_type": "flash_transformer",
  "norm_type": "scalenorm",
  "num_layers": 36,
  "query_key_dim": 128,
  "reduce_group_non_causal_attn": true,
  "shift_tokens": true,
  "torch_dtype": "float32",
  "transformers_version": "4.39.3",
  "vocab_size": 4096
}