Add concurrency

app.py

@@ -1,4 +1,5 @@
 import gradio as gr
+import concurrent.futures
 from haystack.nodes import PromptNode
 
 from utils import lemmatizer_func
@@ -6,8 +7,13 @@ from utils import lemmatizer_func
 def run_prompt(prompt, api_key, model_name, max_length):
     prompt_node = PromptNode(model_name_or_path=model_name, api_key=api_key, max_length=max_length)
     lemmatized_prompt = lemmatizer_func(prompt)
-    response_plain = prompt_node(prompt)
-    response_lemmatized = prompt_node(lemmatized_prompt)
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        future_plain = executor.submit(prompt_node, prompt)
+        future_lemmatized = executor.submit(prompt_node, lemmatized_prompt)
+        
+        response_plain = future_plain.result()
+        response_lemmatized = future_lemmatized.result()
+    
     return lemmatized_prompt, response_plain[0][0], response_plain[1]["prompt_tokens"], response_plain[1]["completion_tokens"], response_lemmatized[0][0], response_lemmatized[1]["prompt_tokens"], response_lemmatized[1]["completion_tokens"]
 
 with gr.Blocks() as demo:
@@ -20,7 +26,7 @@ with gr.Blocks() as demo:
         gr.Examples(
         [
             ["I want you to act as a travel guide. I will write you my location and you will suggest a place to visit near my location. In some cases, I will also give you the type of places I will visit. You will also suggest me places of similar type that are close to my first location. My first suggestion request is \"I am in Italy and I want to visit only museums.\""],
-            ["What's the Everett interpretation of quantum mechanics?"],
+            ["Antibiotics are a type of medication used to treat bacterial infections. They work by either killing the bacteria or preventing them from reproducing, allowing the body’s immune system to fight off the infection. Antibiotics are usually taken orally in the form of pills, capsules, or liquid solutions, or sometimes administered intravenously. They are not effective against viral infections, and using them inappropriately can lead to antibiotic resistance. Explain the above in one sentence:"],
             ["Give me a list of the top 10 dive sites you would recommend around the world."],
             ["Can you tell me more about deep-water soloing?"],
             ["Can you write a short tweet about the Apache 2.0 release of our latest AI model, Falcon LLM?"],