set batch_size to 64
Browse files
model.py
CHANGED
|
@@ -7,7 +7,7 @@ n_threads = os.cpu_count()
|
|
| 7 |
llm = Llama(
|
| 8 |
model_path=model_path,
|
| 9 |
n_ctx=512,
|
| 10 |
-
n_batch=
|
| 11 |
n_threads=n_threads,
|
| 12 |
n_gpu_layers=-1,
|
| 13 |
chat_format="llama-3"
|
|
|
|
| 7 |
llm = Llama(
|
| 8 |
model_path=model_path,
|
| 9 |
n_ctx=512,
|
| 10 |
+
n_batch=64,
|
| 11 |
n_threads=n_threads,
|
| 12 |
n_gpu_layers=-1,
|
| 13 |
chat_format="llama-3"
|