try to fit on T4 (16GB RAM)

app.py

@@ -31,7 +31,9 @@ model = AutoModelForCausalLM.from_pretrained(
     model_name,
     torch_dtype=torch.bfloat16,
     device_map="auto",
-    load_in_8bit=True,
+    # load_in_8bit=True,
+    # try to make it fit on a T4?
+    load_in_4bit=True
 )
 streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)