new commits

app.py

@@ -1,5 +1,6 @@
 from celebbot import CelebBot
 import streamlit as st
+import time
 from streamlit_mic_recorder import speech_to_text
 from utils import *
 
@@ -7,7 +8,7 @@ from utils import *
 def main():
 
     hide_footer()
-    model_list = ["flan-t5-large", "flan-t5-xl", "Falcon-7b-instruct"]
+    model_list = ["flan-t5-xl"]
     celeb_data = get_celeb_data(f'data.json')
 
     st.sidebar.header("CelebChat")
@@ -22,80 +23,86 @@ def main():
         st.session_state["sentTr_model_path"] = "sentence-transformers/all-mpnet-base-v2"
     if "start_chat" not in st.session_state:          
         st.session_state["start_chat"] = False
-    if "prompt" not in st.session_state:          
-        st.session_state["prompt"] = None
-
-    def start_chat(name, model_id):
-        print(name, model_id)
-        if name != '' and model_id != '':
-            st.session_state["start_chat"] = True
-        else:
-            st.session_state["start_chat"] = False
-
-    with st.sidebar.form("my_form"):
-        print("enter form")
-        st.session_state["celeb_name"] = st.selectbox('Choose a celebrity', options=list(celeb_data.keys()))
-        model_id=st.selectbox("Choose Your Flan-T5 model",options=model_list)
-        st.session_state["QA_model_path"] = f"google/{model_id}" if "flan-t5" in model_id else model_id
-
-        st.form_submit_button(label="Start Chatting", on_click=start_chat, args=(st.session_state["celeb_name"], st.session_state["QA_model_path"]))
-
-    if st.session_state["start_chat"]:
-        celeb_gender = celeb_data[st.session_state["celeb_name"]]["gender"]
-        knowledge = celeb_data[st.session_state["celeb_name"]]["knowledge"]
-        st.session_state["celeb_bot"] = CelebBot(st.session_state["celeb_name"], 
-                        get_tokenizer(st.session_state["QA_model_path"]), 
-                        get_seq2seq_model(st.session_state["QA_model_path"]) if "flan-t5" in st.session_state["QA_model_path"] else get_causal_model(st.session_state["QA_model_path"]), 
-                        get_tokenizer(st.session_state["sentTr_model_path"]), 
-                        get_auto_model(st.session_state["sentTr_model_path"]), 
-                        *preprocess_text(st.session_state["celeb_name"], celeb_gender, knowledge, "en_core_web_sm")
-                        )
-
-        dialogue_container = st.container()
-        with dialogue_container:
-            for message in st.session_state["messages"]:
-                with st.chat_message(message["role"]):
-                    st.markdown(message["content"])
+    if "prompt_from_audio" not in st.session_state:          
+        st.session_state["prompt_from_audio"] = ""
+    if "prompt_from_text" not in st.session_state:          
+        st.session_state["prompt_from_text"] = ""
+
+    def text_submit():
+        st.session_state["prompt_from_text"] = st.session_state.widget
+        st.session_state.widget = ''
+
+    st.session_state["celeb_name"] = st.sidebar.selectbox('Choose a celebrity', options=list(celeb_data.keys()))
+    model_id=st.sidebar.selectbox("Choose Your Flan-T5 model",options=model_list)
+    st.session_state["QA_model_path"] = f"google/{model_id}" if "flan-t5" in model_id else model_id
 
+    celeb_gender = celeb_data[st.session_state["celeb_name"]]["gender"]
+    knowledge = celeb_data[st.session_state["celeb_name"]]["knowledge"]
+    st.session_state["celeb_bot"] = CelebBot(st.session_state["celeb_name"], 
+                    get_tokenizer(st.session_state["QA_model_path"]), 
+                    get_seq2seq_model(st.session_state["QA_model_path"]) if "flan-t5" in st.session_state["QA_model_path"] else get_causal_model(st.session_state["QA_model_path"]), 
+                    get_tokenizer(st.session_state["sentTr_model_path"]), 
+                    get_auto_model(st.session_state["sentTr_model_path"]), 
+                    *preprocess_text(st.session_state["celeb_name"], celeb_gender, knowledge, "en_core_web_sm")
+                    )
 
-        if "_last_audio_id" not in st.session_state:
-            st.session_state["_last_audio_id"] = 0
-        with st.sidebar:
-            prompt_from_audio =speech_to_text(start_prompt="Start Recording",stop_prompt="Stop Recording",language='en',use_container_width=True, just_once=True,key='STT')
-            prompt_from_text = st.text_input('Or write something')
+    dialogue_container = st.container()
+    with dialogue_container:
+        for message in st.session_state["messages"]:
+            with st.chat_message(message["role"]):
+                st.markdown(message["content"])
+
+    if "_last_audio_id" not in st.session_state:
+        st.session_state["_last_audio_id"] = 0
+    with st.sidebar:
+        st.session_state["prompt_from_audio"] = speech_to_text(start_prompt="Start Recording",stop_prompt="Stop Recording",language='en',use_container_width=True, just_once=True,key='STT')
+        st.text_input('Or write something', key='widget', on_change=text_submit)
+    
+    if st.session_state["prompt_from_audio"] != None:
+        prompt = st.session_state["prompt_from_audio"] 
+    elif st.session_state["prompt_from_text"] != None:
+        prompt = st.session_state["prompt_from_text"]    
+
+    if prompt != None and prompt != '':
+        st.session_state["celeb_bot"].text = prompt
+        # Display user message in chat message container
+        with dialogue_container:
+            st.chat_message("user").markdown(prompt)
+        # Add user message to chat history
+        st.session_state["messages"].append({"role": "user", "content": prompt})
+
+        # Add assistant response to chat history
+        response = st.session_state["celeb_bot"].question_answer()
         
-        if prompt_from_audio != None:
-            st.session_state["prompt"] = prompt_from_audio 
-        elif prompt_from_text != None:
-            st.session_state["prompt"] = prompt_from_text     
-        print(st.session_state["prompt"])   
-        if st.session_state["prompt"] != None and st.session_state["prompt"] != '':
-            st.session_state["celeb_bot"].text = st.session_state["prompt"]
-            # Display user message in chat message container
-            with dialogue_container:
-                st.chat_message("user").markdown(st.session_state["prompt"])
-            # Add user message to chat history
-            st.session_state["messages"].append({"role": "user", "content": st.session_state["prompt"]})
-
-            # Add assistant response to chat history
-            response = st.session_state["celeb_bot"].question_answer()
-            
-            # disable autoplay to play in HTML
-            b64 = st.session_state["celeb_bot"].text_to_speech(autoplay=False)
-            md = f"""
-            <p>{response}</p>
-            <audio controls autoplay style="display:none;">
-            <source src="data:audio/wav;base64,{b64}" type="audio/wav">
-            Your browser does not support the audio element.
-            </audio>
-            """
-            with dialogue_container:
-                st.chat_message("assistant").markdown(
-                    md,
-                    unsafe_allow_html=True,
-                )
-            # Display assistant response in chat message container
-            st.session_state["messages"].append({"role": "assistant", "content": response})
+        # disable autoplay to play in HTML
+        wav, sr = st.session_state["celeb_bot"].text_to_speech(autoplay=False)
+        md = f"""
+        <p>{response}</p>
+        """
+        with dialogue_container:
+            st.chat_message("assistant").markdown(
+                md,
+                unsafe_allow_html=True,
+            )
+
+        # Play the audio (non-blocking)
+        import sounddevice as sd
+        try:
+            sd.stop()
+            sd.play(wav, sr)
+            time_span = len(wav)//sr + 1
+            time.sleep(time_span)
+
+        except sd.PortAudioError as e:
+            print("\nCaught exception: %s" % repr(e))
+            print("Continuing without audio playback. Suppress this message with the \"--no_sound\" flag.\n")
+        except:
+            raise
+        # Display assistant response in chat message container
+        st.session_state["messages"].append({"role": "assistant", "content": response})
+
+        st.session_state["prompt_from_audio"] = ""   
+        st.session_state["prompt_from_text"] = ""   
 
 
 if __name__ == "__main__":

celebbot.py

@@ -103,12 +103,12 @@ class CelebBot():
             ## have a conversation
             else:
                 if re.search(re.compile(rf'\b(you|your|{self.name})\b', flags=re.IGNORECASE), self.text) != None:              
-                    instruction1 = f'[Instruction] You are a celebrity named {self.name}. You need to answer the question based on knowledge and commonsense.'
+                    instruction1 = f'You are a celebrity named {self.name}. You need to answer the question based on knowledge and commonsense.'
 
                     knowledge = self.retrieve_knowledge_assertions()
                 else:
-                    instruction1 = f'[Instruction] You need to answer the question based on commonsense.'
-                query = f"{instruction1} [knowledge] {knowledge} [question] {self.text}"
+                    instruction1 = f'You need to answer the question based on commonsense.'
+                query = f"Context: {instruction1} {knowledge}\n\nQuestion: {self.text}\n\nAnswer:"
                 input_ids = self.QA_tokenizer(f"{query}", return_tensors="pt").input_ids
                 outputs = self.QA_model.generate(input_ids, max_length=1024)
                 self.text = self.QA_tokenizer.decode(outputs[0], skip_special_tokens=True)

run_tts.py

@@ -109,11 +109,7 @@ def tts(text, embed_name, nlp, autoplay=True):
             print("Continuing without audio playback. Suppress this message with the \"--no_sound\" flag.\n")
         except:
             raise
-    bytes_wav = bytes()
-    byte_io = io.BytesIO(bytes_wav)
-    write(byte_io, synthesizer.sample_rate, wav.astype(np.float32))
-    result_bytes = byte_io.read()
-    return base64.b64encode(result_bytes).decode()
+    return wav, synthesizer.sample_rate
     
 
 if __name__ == "__main__":