Daniel Marques commited on
Commit
0fa1208
1 Parent(s): 3a5d8c7

fix: memory error

Browse files
Files changed (2) hide show
  1. constants.py +2 -2
  2. load_models.py +1 -1
constants.py CHANGED
@@ -32,13 +32,13 @@ CHROMA_SETTINGS = Settings(
32
  )
33
 
34
  # Context Window and Max New Tokens
35
- CONTEXT_WINDOW_SIZE = 4096
36
  MAX_NEW_TOKENS = CONTEXT_WINDOW_SIZE # int(CONTEXT_WINDOW_SIZE/4)
37
 
38
  #### If you get a "not enough space in the buffer" error, you should reduce the values below, start with half of the original values and keep halving the value until the error stops appearing
39
 
40
  N_GPU_LAYERS = 100 # Llama-2-70B has 83 layers
41
- N_BATCH = 512
42
 
43
  ### From experimenting with the Llama-2-7B-Chat-GGML model on 8GB VRAM, these values work:
44
  # N_GPU_LAYERS = 20
 
32
  )
33
 
34
  # Context Window and Max New Tokens
35
+ CONTEXT_WINDOW_SIZE = 2048
36
  MAX_NEW_TOKENS = CONTEXT_WINDOW_SIZE # int(CONTEXT_WINDOW_SIZE/4)
37
 
38
  #### If you get a "not enough space in the buffer" error, you should reduce the values below, start with half of the original values and keep halving the value until the error stops appearing
39
 
40
  N_GPU_LAYERS = 100 # Llama-2-70B has 83 layers
41
+ N_BATCH = 1024
42
 
43
  ### From experimenting with the Llama-2-7B-Chat-GGML model on 8GB VRAM, these values work:
44
  # N_GPU_LAYERS = 20
load_models.py CHANGED
@@ -6,7 +6,7 @@ from auto_gptq import AutoGPTQForCausalLM
6
  from huggingface_hub import hf_hub_download
7
  from langchain.llms import LlamaCpp, HuggingFacePipeline
8
 
9
- from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
10
 
11
  from transformers import (
12
  AutoModelForCausalLM,
 
6
  from huggingface_hub import hf_hub_download
7
  from langchain.llms import LlamaCpp, HuggingFacePipeline
8
 
9
+ # from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
10
 
11
  from transformers import (
12
  AutoModelForCausalLM,