Update README.md
Browse files
README.md
CHANGED
|
@@ -12,6 +12,22 @@ tags:
|
|
| 12 |
---
|
| 13 |
# OpenVINO IR model with int8 quantization of Starling-LM-7B-beta
|
| 14 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
<!-- Provide a quick summary of what the model is/does. -->
|
| 16 |
|
| 17 |
|
|
|
|
| 12 |
---
|
| 13 |
# OpenVINO IR model with int8 quantization of Starling-LM-7B-beta
|
| 14 |
|
| 15 |
+
Model definition for LocalAI
|
| 16 |
+
```
|
| 17 |
+
name: starling-lm-7b-beta
|
| 18 |
+
backend: transformers
|
| 19 |
+
parameters:
|
| 20 |
+
model: fakezeta/Starling-LM-7B-beta-openvino-int8
|
| 21 |
+
context_size: 8192
|
| 22 |
+
threads: 6
|
| 23 |
+
f16: true
|
| 24 |
+
type: OVModelForCausalLM
|
| 25 |
+
prompt_cache_path: "cache"
|
| 26 |
+
prompt_cache_all: true
|
| 27 |
+
template:
|
| 28 |
+
use_tokenizer_template: true
|
| 29 |
+
```
|
| 30 |
+
|
| 31 |
<!-- Provide a quick summary of what the model is/does. -->
|
| 32 |
|
| 33 |
|