[MOD] Quantizied Carballo-Cerebras

app.py

@@ -1,11 +1,11 @@
 import os
 import gradio as gr
 from gradio.components import Slider
+from transformers import pipeline, AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
 import torch
-from transformers import pipeline
 
 # Model, information and examples ----------------------------------------------
-MODEL_NAMES = ["Carballo-bloom-1.3B","Carballo-cerebras-1.3B","Carvalho_pt-gl"]
+MODEL_NAMES = ["Carballo-bloom-1.3B","Carvalho_pt-gl","Carballo-cerebras-1.3B"]
 markdown_description_gl = """
 [*English below*]
 
@@ -57,11 +57,21 @@ fronted_theme = 'Soft'
 # Model charge ---------------------------------------------------------
 model_id_bloom  = "proxectonos/Carballo-bloom-1.3B"
 generator_model_bloom = pipeline("text-generation", model=model_id_bloom)
-model_id_cerebras  = "proxectonos/Carballo-cerebras-1.3B"
-generator_model_cerebras = pipeline("text-generation", model=model_id_cerebras)
 model_id_carvalho = "Nos-PT/Carvalho_pt-gl-1.3B"
 generator_model_carvalho = pipeline("text-generation", model=model_id_carvalho)
 
+#Quantized Carballo-Cerebras (model less used) to save memory 
+model_id_cerebras  = "proxectonos/Carballo-cerebras-1.3B"
+quantization_config = BitsAndBytesConfig(
+        load_in_4bit=True,
+        bnb_4bit_quant_type="nf4",
+        bnb_4bit_compute_dtype=torch.float16,
+)
+
+model_quantizied = AutoModelForCausalLM.from_pretrained(model_id_cerebras, quantization_config=quantization_config)
+tokenizer = AutoTokenizer.from_pretrained(model_id_cerebras)
+generator_model_cerebras = pipeline("text-generation", model=model_id_cerebras)
+
 # Generation functions ---------------------------------------------------------
 def get_model(model_selection):
     if model_selection == "Carballo-bloom-1.3B":