Hugging Face's logo

aws-neuron
/
optimum-neuron-cache

Model card Files Community
180
dacorvo HF staff commited on
Commit
ee458f5
•
1 Parent(s): 663d611

Update Mistral cached configurations

Browse files
Files changed (1) hide show
  1. inference-cache-config/mistral.json +14 -42
inference-cache-config/mistral.json CHANGED
@@ -1,90 +1,62 @@
1
  {
2
- "mistralai/Mistral-7B-Instruct-v0.2": [
3
  {
4
  "batch_size": 1,
5
  "sequence_length": 4096,
6
  "num_cores": 2,
7
- "auto_cast_type": "bf16"
8
  },
9
  {
10
  "batch_size": 1,
11
  "sequence_length": 4096,
12
  "num_cores": 8,
13
- "auto_cast_type": "bf16"
14
  },
15
  {
16
  "batch_size": 4,
17
  "sequence_length": 4096,
18
  "num_cores": 2,
19
- "auto_cast_type": "bf16"
20
  },
21
  {
22
  "batch_size": 4,
23
  "sequence_length": 4096,
24
  "num_cores": 8,
25
- "auto_cast_type": "bf16"
26
  },
27
  {
28
  "batch_size": 8,
29
  "sequence_length": 4096,
30
  "num_cores": 8,
31
- "auto_cast_type": "bf16"
32
  },
33
  {
34
  "batch_size": 16,
35
  "sequence_length": 4096,
36
  "num_cores": 8,
37
- "auto_cast_type": "bf16"
38
  },
39
  {
40
  "batch_size": 32,
41
  "sequence_length": 4096,
42
  "num_cores": 8,
43
- "auto_cast_type": "bf16"
44
  }
45
  ],
46
- "mistralai/Mistral-7B-Instruct-v0.1": [
47
  {
48
  "batch_size": 1,
49
  "sequence_length": 4096,
50
  "num_cores": 2,
51
- "auto_cast_type": "bf16"
52
- },
 
 
53
  {
54
  "batch_size": 1,
55
  "sequence_length": 4096,
56
- "num_cores": 8,
57
- "auto_cast_type": "bf16"
58
- },
59
- {
60
- "batch_size": 4,
61
- "sequence_length": 4096,
62
  "num_cores": 2,
63
- "auto_cast_type": "bf16"
64
- },
65
- {
66
- "batch_size": 4,
67
- "sequence_length": 4096,
68
- "num_cores": 8,
69
- "auto_cast_type": "bf16"
70
- },
71
- {
72
- "batch_size": 8,
73
- "sequence_length": 4096,
74
- "num_cores": 8,
75
- "auto_cast_type": "bf16"
76
- },
77
- {
78
- "batch_size": 16,
79
- "sequence_length": 4096,
80
- "num_cores": 8,
81
- "auto_cast_type": "bf16"
82
- },
83
- {
84
- "batch_size": 32,
85
- "sequence_length": 4096,
86
- "num_cores": 8,
87
- "auto_cast_type": "bf16"
88
  }
89
  ]
90
  }
 
1
  {
2
+ "mistralai/Mistral-7B-Instruct-v0.3": [
3
  {
4
  "batch_size": 1,
5
  "sequence_length": 4096,
6
  "num_cores": 2,
7
+ "auto_cast_type": "fp16"
8
  },
9
  {
10
  "batch_size": 1,
11
  "sequence_length": 4096,
12
  "num_cores": 8,
13
+ "auto_cast_type": "fp16"
14
  },
15
  {
16
  "batch_size": 4,
17
  "sequence_length": 4096,
18
  "num_cores": 2,
19
+ "auto_cast_type": "fp16"
20
  },
21
  {
22
  "batch_size": 4,
23
  "sequence_length": 4096,
24
  "num_cores": 8,
25
+ "auto_cast_type": "fp16"
26
  },
27
  {
28
  "batch_size": 8,
29
  "sequence_length": 4096,
30
  "num_cores": 8,
31
+ "auto_cast_type": "fp16"
32
  },
33
  {
34
  "batch_size": 16,
35
  "sequence_length": 4096,
36
  "num_cores": 8,
37
+ "auto_cast_type": "fp16"
38
  },
39
  {
40
  "batch_size": 32,
41
  "sequence_length": 4096,
42
  "num_cores": 8,
43
+ "auto_cast_type": "fp16"
44
  }
45
  ],
46
+ "mistralai/Mistral-7B-Instruct-v0.2": [
47
  {
48
  "batch_size": 1,
49
  "sequence_length": 4096,
50
  "num_cores": 2,
51
+ "auto_cast_type": "fp16"
52
+ }
53
+ ],
54
+ "mistralai/Mistral-7B-Instruct-v0.1": [
55
  {
56
  "batch_size": 1,
57
  "sequence_length": 4096,
 
 
 
 
 
 
58
  "num_cores": 2,
59
+ "auto_cast_type": "fp16"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
60
  }
61
  ]
62
  }