Add application file

app.py

@@ -0,0 +1,59 @@
+import gradio as gr
+import torch
+
+from transformers import T5Tokenizer, T5ForConditionalGeneration
+
+# Define hyperparameters
+max_seq_length = 512
+max_output_length = 1024
+num_beams = 16
+length_penalty = 1.4
+no_repeat_ngram_size = 2
+temperature = 0.7
+top_k = 150
+top_p = 0.92
+repetition_penalty = 2.1
+early_stopping = True
+
+# Load the pre-trained model and tokenizer
+model_name = "google/flan-t5-large"
+tokenizer = T5Tokenizer.from_pretrained(model_name, model_max_length=512)
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+if torch.cuda.device_count() > 1:
+    device_ids = [i for i in range(torch.cuda.device_count())]
+    model = torch.nn.DataParallel(T5ForConditionalGeneration.from_pretrained(model_name, return_dict=True), device_ids=device_ids)
+else:
+    model = T5ForConditionalGeneration.from_pretrained(model_name, return_dict=True)
+
+model.to(device)
+
+# Define a function to generate a response to user input
+def chatbot(text):
+    with torch.no_grad():
+        # Tokenize the input text and convert to a PyTorch tensor
+        input_ids = tokenizer(text, return_tensors="pt", padding=True, truncation=True, max_length=max_seq_length).input_ids.to(device)
+
+        # Generate a response using the model
+        if torch.cuda.device_count() > 1:
+            outputs = model.module.generate(input_ids, min_length=max_seq_length, max_new_tokens=max_output_length, num_beams=num_beams, length_penalty=length_penalty, no_repeat_ngram_size=no_repeat_ngram_size, temperature=temperature, top_k=top_k, top_p=top_p, repetition_penalty=repetition_penalty, early_stopping=early_stopping)
+        else:
+            outputs = model.generate(input_ids, min_length=max_seq_length, max_new_tokens=max_output_length, num_beams=num_beams, length_penalty=length_penalty, no_repeat_ngram_size=no_repeat_ngram_size, temperature=temperature, top_k=top_k, top_p=top_p, repetition_penalty=repetition_penalty, early_stopping=early_stopping)
+
+        # Decode the response and return it
+        response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+        return response
+
+chat_help_text = "Welcome! This ChatBot is designed to answer questions about a wide range of topics. " \
+                 "Please note that the ChatBot may not always provide accurate or complete answers, and may not " \
+                 "understand certain questions. To use the ChatBot, simply type in your question in the text box " \
+                 "below and hit Enter or click the button. Please keep in mind that the ChatBot is not perfect " \
+                 "and may provide inaccurate or incomplete answers. It is best suited for simple factual " \
+                 "questions rather than complex or nuanced inquiries."
+
+# Create a Gradio interface
+iface = gr.Interface(fn=chatbot, inputs="text", outputs="text", title="NuNet Inferencing Demo",
+                     description=chat_help_text)
+
+iface.launch(share=True)
+