GGUF
Generated from Trainer
axolotl
llama-cpp
gguf-my-repo
LMEngine
tinybiggames commited on
Commit
65dfba0
1 Parent(s): a86f565

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +12 -12
README.md CHANGED
@@ -24,13 +24,13 @@ model-index:
24
  # tinybiggames/dolphin-2.9.1-llama-3-8b-Q4_K_M-GGUF
25
  This model was converted to GGUF format from [`cognitivecomputations/dolphin-2.9.1-llama-3-8b`](https://huggingface.co/cognitivecomputations/dolphin-2.9.1-llama-3-8b) using llama.cpp via the ggml.ai's [GGUF-my-repo](https://huggingface.co/spaces/ggml-org/gguf-my-repo) space.
26
  Refer to the [original model card](https://huggingface.co/cognitivecomputations/dolphin-2.9.1-llama-3-8b) for more details on the model.
27
- ## Use with tinyBigGAMES's [LMEngine Inference Library](https://github.com/tinyBigGAMES/LMEngine)
28
 
29
 
30
  How to configure LMEngine:
31
 
32
  ```Delphi
33
- LME_InitConfig(
34
  'C:/LLM/gguf', // path to model files
35
  -1 // number of GPU layer, -1 to use all available layers
36
  );
@@ -39,7 +39,7 @@ LME_InitConfig(
39
  How to define model:
40
 
41
  ```Delphi
42
- LME_DefineModel('dolphin-2.9.1-llama-3-8b.Q4_K_M.gguf',
43
  'dolphin-2.9.1-llama-3-8b.Q4_K_M', 8000,
44
  '<|im_start|>{role}\n{content}<|im_end|>\n',
45
  '<|im_start|>assistant');
@@ -48,8 +48,8 @@ LME_DefineModel('dolphin-2.9.1-llama-3-8b.Q4_K_M.gguf',
48
  How to add a message:
49
 
50
  ```Delphi
51
- LME_AddMessage(
52
- LME_ROLE_USER, // role
53
  'What is AI?' // content
54
  );
55
  ```
@@ -66,17 +66,17 @@ var
66
  LOutputTokens: Int32;
67
  LTotalTokens: Int32;
68
 
69
- if LME_RunInference('dolphin-2.9.1-llama-3-8b.Q4_K_M', 1024) then
70
  begin
71
- LME_GetInferenceStats(nil, @LTokenOutputSpeed, @LInputTokens, @LOutputTokens,
72
  @LTotalTokens);
73
- LME_PrintLn('', LME_FG_WHITE);
74
- LME_PrintLn('Tokens :: Input: %d, Output: %d, Total: %d, Speed: %3.1f t/s',
75
- LME_FG_BRIGHTYELLOW, LInputTokens, LOutputTokens, LTotalTokens, LTokenOutputSpeed);
76
  end
77
  else
78
  begin
79
- LME_PrintLn('', LME_FG_WHITE);
80
- LME_PrintLn('Error: %s', LME_FG_RED, LME_GetError());
81
  end;
82
  ```
 
24
  # tinybiggames/dolphin-2.9.1-llama-3-8b-Q4_K_M-GGUF
25
  This model was converted to GGUF format from [`cognitivecomputations/dolphin-2.9.1-llama-3-8b`](https://huggingface.co/cognitivecomputations/dolphin-2.9.1-llama-3-8b) using llama.cpp via the ggml.ai's [GGUF-my-repo](https://huggingface.co/spaces/ggml-org/gguf-my-repo) space.
26
  Refer to the [original model card](https://huggingface.co/cognitivecomputations/dolphin-2.9.1-llama-3-8b) for more details on the model.
27
+ ## Use with tinyBigGAMES's [Inference](https://github.com/tinyBigGAMES) Libraries.
28
 
29
 
30
  How to configure LMEngine:
31
 
32
  ```Delphi
33
+ InitConfig(
34
  'C:/LLM/gguf', // path to model files
35
  -1 // number of GPU layer, -1 to use all available layers
36
  );
 
39
  How to define model:
40
 
41
  ```Delphi
42
+ DefineModel('dolphin-2.9.1-llama-3-8b.Q4_K_M.gguf',
43
  'dolphin-2.9.1-llama-3-8b.Q4_K_M', 8000,
44
  '<|im_start|>{role}\n{content}<|im_end|>\n',
45
  '<|im_start|>assistant');
 
48
  How to add a message:
49
 
50
  ```Delphi
51
+ AddMessage(
52
+ ROLE_USER, // role
53
  'What is AI?' // content
54
  );
55
  ```
 
66
  LOutputTokens: Int32;
67
  LTotalTokens: Int32;
68
 
69
+ if RunInference('dolphin-2.9.1-llama-3-8b.Q4_K_M', 1024) then
70
  begin
71
+ GetInferenceStats(nil, @LTokenOutputSpeed, @LInputTokens, @LOutputTokens,
72
  @LTotalTokens);
73
+ PrintLn('', FG_WHITE);
74
+ PrintLn('Tokens :: Input: %d, Output: %d, Total: %d, Speed: %3.1f t/s',
75
+ FG_BRIGHTYELLOW, LInputTokens, LOutputTokens, LTotalTokens, LTokenOutputSpeed);
76
  end
77
  else
78
  begin
79
+ PrintLn('', FG_WHITE);
80
+ PrintLn('Error: %s', FG_RED, GetError());
81
  end;
82
  ```