saattrupdan commited on
Commit
0bf024b
·
verified ·
1 Parent(s): cb18c3c

Change dtype to bf16

Browse files

This enables vLLM to load the model in bf16 by default.

Files changed (1) hide show
  1. config.json +1 -1
config.json CHANGED
@@ -29,7 +29,7 @@
29
  },
30
  "rope_theta": 500000.0,
31
  "tie_word_embeddings": false,
32
- "torch_dtype": "float32",
33
  "transformers_version": "4.46.3",
34
  "use_cache": true,
35
  "vocab_size": 128256
 
29
  },
30
  "rope_theta": 500000.0,
31
  "tie_word_embeddings": false,
32
+ "torch_dtype": "bfloat16",
33
  "transformers_version": "4.46.3",
34
  "use_cache": true,
35
  "vocab_size": 128256