dacorvo HF staff commited on
Commit
57652e6
1 Parent(s): 8761969

Create mixtral.json

Browse files
Files changed (1) hide show
  1. inference-cache-config/mixtral.json +16 -0
inference-cache-config/mixtral.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "mistralai/Mixtral-8x7B-Instruct-v0.1": [
3
+ {
4
+ "batch_size": 1,
5
+ "sequence_length": 4096,
6
+ "num_cores": 24,
7
+ "auto_cast_type": "fp16"
8
+ },
9
+ {
10
+ "batch_size": 4,
11
+ "sequence_length": 4096,
12
+ "num_cores": 24,
13
+ "auto_cast_type": "fp16"
14
+ }
15
+ ]
16
+ }