tinybiggames
commited on
Commit
•
65dfba0
1
Parent(s):
a86f565
Update README.md
Browse files
README.md
CHANGED
@@ -24,13 +24,13 @@ model-index:
|
|
24 |
# tinybiggames/dolphin-2.9.1-llama-3-8b-Q4_K_M-GGUF
|
25 |
This model was converted to GGUF format from [`cognitivecomputations/dolphin-2.9.1-llama-3-8b`](https://huggingface.co/cognitivecomputations/dolphin-2.9.1-llama-3-8b) using llama.cpp via the ggml.ai's [GGUF-my-repo](https://huggingface.co/spaces/ggml-org/gguf-my-repo) space.
|
26 |
Refer to the [original model card](https://huggingface.co/cognitivecomputations/dolphin-2.9.1-llama-3-8b) for more details on the model.
|
27 |
-
## Use with tinyBigGAMES's [
|
28 |
|
29 |
|
30 |
How to configure LMEngine:
|
31 |
|
32 |
```Delphi
|
33 |
-
|
34 |
'C:/LLM/gguf', // path to model files
|
35 |
-1 // number of GPU layer, -1 to use all available layers
|
36 |
);
|
@@ -39,7 +39,7 @@ LME_InitConfig(
|
|
39 |
How to define model:
|
40 |
|
41 |
```Delphi
|
42 |
-
|
43 |
'dolphin-2.9.1-llama-3-8b.Q4_K_M', 8000,
|
44 |
'<|im_start|>{role}\n{content}<|im_end|>\n',
|
45 |
'<|im_start|>assistant');
|
@@ -48,8 +48,8 @@ LME_DefineModel('dolphin-2.9.1-llama-3-8b.Q4_K_M.gguf',
|
|
48 |
How to add a message:
|
49 |
|
50 |
```Delphi
|
51 |
-
|
52 |
-
|
53 |
'What is AI?' // content
|
54 |
);
|
55 |
```
|
@@ -66,17 +66,17 @@ var
|
|
66 |
LOutputTokens: Int32;
|
67 |
LTotalTokens: Int32;
|
68 |
|
69 |
-
if
|
70 |
begin
|
71 |
-
|
72 |
@LTotalTokens);
|
73 |
-
|
74 |
-
|
75 |
-
|
76 |
end
|
77 |
else
|
78 |
begin
|
79 |
-
|
80 |
-
|
81 |
end;
|
82 |
```
|
|
|
24 |
# tinybiggames/dolphin-2.9.1-llama-3-8b-Q4_K_M-GGUF
|
25 |
This model was converted to GGUF format from [`cognitivecomputations/dolphin-2.9.1-llama-3-8b`](https://huggingface.co/cognitivecomputations/dolphin-2.9.1-llama-3-8b) using llama.cpp via the ggml.ai's [GGUF-my-repo](https://huggingface.co/spaces/ggml-org/gguf-my-repo) space.
|
26 |
Refer to the [original model card](https://huggingface.co/cognitivecomputations/dolphin-2.9.1-llama-3-8b) for more details on the model.
|
27 |
+
## Use with tinyBigGAMES's [Inference](https://github.com/tinyBigGAMES) Libraries.
|
28 |
|
29 |
|
30 |
How to configure LMEngine:
|
31 |
|
32 |
```Delphi
|
33 |
+
InitConfig(
|
34 |
'C:/LLM/gguf', // path to model files
|
35 |
-1 // number of GPU layer, -1 to use all available layers
|
36 |
);
|
|
|
39 |
How to define model:
|
40 |
|
41 |
```Delphi
|
42 |
+
DefineModel('dolphin-2.9.1-llama-3-8b.Q4_K_M.gguf',
|
43 |
'dolphin-2.9.1-llama-3-8b.Q4_K_M', 8000,
|
44 |
'<|im_start|>{role}\n{content}<|im_end|>\n',
|
45 |
'<|im_start|>assistant');
|
|
|
48 |
How to add a message:
|
49 |
|
50 |
```Delphi
|
51 |
+
AddMessage(
|
52 |
+
ROLE_USER, // role
|
53 |
'What is AI?' // content
|
54 |
);
|
55 |
```
|
|
|
66 |
LOutputTokens: Int32;
|
67 |
LTotalTokens: Int32;
|
68 |
|
69 |
+
if RunInference('dolphin-2.9.1-llama-3-8b.Q4_K_M', 1024) then
|
70 |
begin
|
71 |
+
GetInferenceStats(nil, @LTokenOutputSpeed, @LInputTokens, @LOutputTokens,
|
72 |
@LTotalTokens);
|
73 |
+
PrintLn('', FG_WHITE);
|
74 |
+
PrintLn('Tokens :: Input: %d, Output: %d, Total: %d, Speed: %3.1f t/s',
|
75 |
+
FG_BRIGHTYELLOW, LInputTokens, LOutputTokens, LTotalTokens, LTokenOutputSpeed);
|
76 |
end
|
77 |
else
|
78 |
begin
|
79 |
+
PrintLn('', FG_WHITE);
|
80 |
+
PrintLn('Error: %s', FG_RED, GetError());
|
81 |
end;
|
82 |
```
|