Change to use GPTQ so i dont get limited

app.py

@@ -1,7 +1,7 @@
 import gradio as gr
 from huggingface_hub import InferenceClient
 from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
-from transformers import BitsAndBytesConfig
+from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig
 import spaces
 import torch
 from safetensors import safe_open
@@ -11,10 +11,18 @@ from torch import Tensor
 from threading import Thread
 import einops
 
-
-tokenizer = AutoTokenizer.from_pretrained("NousResearch/Meta-LLaMA-3-70B-Instruct")
-quantization_config = BitsAndBytesConfig(load_in_4_bit=True)
-model = AutoModelForCausalLM.from_pretrained("NousResearch/Meta-LLaMA-3-70B-Instruct", quantization_config, device_map="cuda" ).eval()
+model_id = "MaziyarPanahi/Meta-Llama-3-70B-Instruct-GPTQ"
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+quantize_config = BaseQuantizeConfig(
+        bits=4,
+        group_size=128,
+        desc_act=False
+    )
+model = AutoGPTQForCausalLM.from_quantized(
+        model_id,
+        use_safetensors=True,
+        device="cuda",
+        quantize_config=quantize_config).eval()
 
 
 @spaces.GPU