Update app.py

app.py

@@ -16,7 +16,7 @@ ROLE_TOKENS = {
     "system": SYSTEM_TOKEN
 }
 
-CONTEXT_SIZE = 4096
+CONTEXT_SIZE = 3000
 
 # Create a lock object
 lock = threading.Lock()
@@ -47,7 +47,7 @@ model = Llama(
     logits_all=True,
     #n_threads=12,
     verbose=True,
-    n_gpu_layers=30,
+    n_gpu_layers=35,
     n_gqa=8       #must be set for 70b models
 )