Update app.py

app.py

@@ -16,18 +16,13 @@ MODEL_ID1 = "microsoft/Phi-3.5-mini-instruct"
 MODEL_LIST1 = ["microsoft/Phi-3.5-mini-instruct"]
 HF_TOKEN = os.environ.get("HF_TOKEN", None)
 
-device = "cuda" if torch.cuda.is_available() else "cpu"
-
-# Quantisierungskonfiguration nur aktivieren, wenn eine GPU vorhanden ist
-if device == "cuda":
-    quantization_config = BitsAndBytesConfig(
-        load_in_4bit=True,
-        bnb_4bit_compute_dtype=torch.bfloat16,
-        bnb_4bit_use_double_quant=True,
-        bnb_4bit_quant_type="nf4"
-    )
-else:
-    quantization_config = None  # Keine Quantisierung auf CPU
+device = "cuda" if torch.cuda.is_available() else "cpu"  # for GPU usage or "cpu" for CPU usage / But you need GPU :)
+
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_compute_dtype=torch.bfloat16,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4")
 
 tokenizer = AutoTokenizer.from_pretrained(MODEL_ID1)
 model = AutoModelForCausalLM.from_pretrained(
@@ -280,4 +275,4 @@ with gr.Blocks(css=CSS, theme="small_and_pretty") as demo:
     gr.HTML(footer)
 
 # Launch the combined app
-demo.launch(debug=True)
+demo.launch(debug=True)