FP 16

app.py

@@ -11,10 +11,10 @@ import gradio as gr
 
 device = "cuda:0" if torch.cuda.is_available() else "cpu"
 
-"""##FP 32"""
+"""##FP 16"""
 
 tokenizer = T5Tokenizer.from_pretrained("google/flan-t5-xl")
-model = T5ForConditionalGeneration.from_pretrained("google/flan-t5-xl", device_map="auto")
+model = T5ForConditionalGeneration.from_pretrained("google/flan-t5-xl", device_map="auto", torch_dtype=torch.float16)
 
 """###Interface"""
 
@@ -24,7 +24,7 @@ def generate(input_text, minimum_length, maximum_length, temperature, repetition
                           min_length=minimum_length,
                           max_new_tokens=maximum_length,
                           length_penalty=1.4,
-                          num_beams=6,
+                          num_beams=12,
                           no_repeat_ngram_size=3,
                           temperature=temperature,
                           top_k=100,
@@ -34,14 +34,14 @@ def generate(input_text, minimum_length, maximum_length, temperature, repetition
 
   return tokenizer.decode(outputs[0], skip_special_tokens=True).capitalize()
 
-title = "Flan-T5-XL Inference on GRADIO GUI"
+title = "Flan-T5-XL GRADIO GUI"
 
 def inference(input_text, minimum_length, maximum_length, temperature, repetition_penalty):
   return generate(input_text, minimum_length, maximum_length, temperature, repetition_penalty)
 
 gr.Interface(
     fn=inference,
-    inputs=[gr.Textbox(lines=4, label="Input text"), gr.Slider(0, 300, value=20, step=10, label="Minimum length"), gr.Slider(100, 2000, value=1000, step=100, label="Maximum length"), gr.Slider(0, 2, value=0.8, step=0.1, label="Temperature"), gr.Slider(1, 3, value=2.1, step=0.1, label="Repetition penalty")],
+    inputs=[gr.Textbox(lines=4, label="Input text"), gr.Slider(0, 300, value=20, step=10, label="Minimum length"), gr.Slider(100, 2000, value=1000, step=100, label="Maximum length"), gr.Slider(0, 2, value=0.7, step=0.1, label="Temperature"), gr.Slider(1, 3, value=2.1, step=0.1, label="Repetition penalty")],
     outputs=[
         gr.Textbox(lines=2, label="Flan-T5-XL Inference")
     ],