Update app.py

app.py

@@ -2,7 +2,7 @@ import os
 import time
 import spaces
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig
 import gradio as gr
 from threading import Thread
 
@@ -33,13 +33,14 @@ h3 {
 
 device = "cuda" # for GPU usage or "cpu" for CPU usage
 
+quantization_config = BitsAndBytesConfig(load_in_8bit=True)
+
 tokenizer = AutoTokenizer.from_pretrained(MODEL)
 model = AutoModelForCausalLM.from_pretrained(
     MODEL,
     torch_dtype=torch.bfloat16,
-    low_cpu_mem_usage=True,
     device_map="auto",
-    trust_remote_code=True)
+    quantization_config=quantization_config)
 
 @spaces.GPU()
 def stream_chat(