Daniel Marques commited on
Commit
ba8f696
1 Parent(s): de59564

fix: memory error

Browse files
Files changed (1) hide show
  1. constants.py +3 -3
constants.py CHANGED
@@ -32,13 +32,13 @@ CHROMA_SETTINGS = Settings(
32
  )
33
 
34
  # Context Window and Max New Tokens
35
- CONTEXT_WINDOW_SIZE = 2000
36
  MAX_NEW_TOKENS = CONTEXT_WINDOW_SIZE # int(CONTEXT_WINDOW_SIZE/4)
37
 
38
  #### If you get a "not enough space in the buffer" error, you should reduce the values below, start with half of the original values and keep halving the value until the error stops appearing
39
 
40
- N_GPU_LAYERS = 83 # Llama-2-70B has 83 layers
41
- N_BATCH = MAX_NEW_TOKENS
42
 
43
  ### From experimenting with the Llama-2-7B-Chat-GGML model on 8GB VRAM, these values work:
44
  # N_GPU_LAYERS = 20
 
32
  )
33
 
34
  # Context Window and Max New Tokens
35
+ CONTEXT_WINDOW_SIZE = 4096
36
  MAX_NEW_TOKENS = CONTEXT_WINDOW_SIZE # int(CONTEXT_WINDOW_SIZE/4)
37
 
38
  #### If you get a "not enough space in the buffer" error, you should reduce the values below, start with half of the original values and keep halving the value until the error stops appearing
39
 
40
+ N_GPU_LAYERS = 100 # Llama-2-70B has 83 layers
41
+ N_BATCH = 512
42
 
43
  ### From experimenting with the Llama-2-7B-Chat-GGML model on 8GB VRAM, these values work:
44
  # N_GPU_LAYERS = 20