refactor - move ResponseState to responce_state_manager

charles_app.py

@@ -4,7 +4,7 @@ import time
 import asyncio
 import os
 from clip_transform import CLIPTransform
-from responce_state_manager import ResponceStateManager
+from response_state_manager import ResponseStateManager
 from respond_to_prompt_async import RespondToPromptAsync
 import asyncio
 import subprocess
@@ -32,8 +32,8 @@ class CharlesApp:
         self._app_interface_actor = AppInterfaceActor.get_singleton()
         self._audio_output_queue = await self._app_interface_actor.get_audio_output_queue.remote()
 
-        self.set_state("002 - creating ResponceStateManager")
-        self._responce_state_manager = ResponceStateManager()
+        self.set_state("002 - creating ResponseStateManager")
+        self._response_state_manager = ResponseStateManager()
 
         self.set_state("003 - creating PromptManager")
         from prompt_manager import PromptManager
@@ -88,8 +88,6 @@ class CharlesApp:
         vector_debug = "--n/a--"
         
         process_speech_to_text_future = []
-        current_responses = []
-        speech_chunks_per_response = []
         human_preview_text = ""
         robot_preview_text = ""
         additional_prompt = None
@@ -98,7 +96,7 @@ class CharlesApp:
         has_spoken_for_this_prompt = False
 
         while True:
-            responce_step = self._responce_state_manager.begin_next_step()
+            response_step_obs, response_state = self._response_state_manager.begin_next_step()
             audio_frames = await self._app_interface_actor.dequeue_audio_input_frames_async.remote()    
             video_frames = await self._app_interface_actor.dequeue_video_input_frames_async.remote()
 
@@ -131,10 +129,9 @@ class CharlesApp:
                     if speaker_finished and len(prompt) > 0 and prompt not in prompts_to_ignore:
                         print(f"Prompt: {prompt}")
                         line = ""
-                        for i, response in enumerate(current_responses):
+                        for i, response in enumerate(response_state.current_responses):
                             line += "🤖 " if len(line) == 0 else ""
-                            # line += f"{response} [{speech_chunks_per_response[i]}]  \n"
-                            line += f"[{speech_chunks_per_response[i]}] {response}  \n"
+                            line += f"[{response_state.speech_chunks_per_response[i]}] {response}  \n"
                         if len(line) > 0:
                             await add_debug_output(line)
                         human_preview_text = ""
@@ -146,15 +143,13 @@ class CharlesApp:
                         if self._respond_to_prompt_task is not None:
                             await self._respond_to_prompt.terminate()
                             self._respond_to_prompt_task.cancel()
-                        self._respond_to_prompt = RespondToPromptAsync(self._responce_state_manager, self._audio_output_queue)
+                        self._respond_to_prompt = RespondToPromptAsync(self._response_state_manager, self._audio_output_queue)
                         self._respond_to_prompt_task = asyncio.create_task(self._respond_to_prompt.run(prompt, self._prompt_manager.messages))
                         additional_prompt = None
                         previous_prompt = prompt
                         is_talking = False
                         has_spoken_for_this_prompt = False
-                        responce_step = self._responce_state_manager.reset_episode()
-                        current_responses = []
-                        speech_chunks_per_response = []
+                        response_step_obs, response_state = self._response_state_manager.reset_episode()
                     elif len(prompt) > 0 and prompt not in prompts_to_ignore:
                         # sometimes we get a false signal of speaker_finsihed
                         # in which case we get new prompts before we have spoken
@@ -166,34 +161,23 @@ class CharlesApp:
                                 self._respond_to_prompt_task.cancel()
                             self._respond_to_prompt_task = None
                             self._respond_to_prompt = None
-                            responce_step = self._responce_state_manager.reset_episode()
-                            current_responses = []
-                            speech_chunks_per_response = []                            
+                            response_step_obs, response_state = self._response_state_manager.reset_episode()                        
                         if additional_prompt is not None:
                             prompt = additional_prompt + ". " + prompt                        
                         human_preview_text = f"👨❓ {prompt}"
 
-            for new_response in responce_step.llm_responses:
+            for new_response in response_step_obs.llm_responses:
                 # add_debug_output(f"🤖 {new_response}")
                 self._prompt_manager.append_assistant_message(new_response)
-                current_responses.append(new_response)
-                speech_chunks_per_response.append(0)
                 robot_preview_text = ""
-            if len(responce_step.llm_preview):
-                robot_preview_text = f"🤖❓ {responce_step.llm_preview}"
-
-            for chunk in responce_step.tts_raw_chunk_ids:
-                chunk = json.loads(chunk)
-                # prompt = chunk['prompt']
-                response_id = chunk['llm_sentence_id']
-                speech_chunks_per_response[response_id] += 1
+            if len(response_step_obs.llm_preview):
+                robot_preview_text = f"🤖❓ {response_step_obs.llm_preview}"
 
             list_of_strings = debug_output_history.copy()
             line = ""
-            for i, response in enumerate(current_responses):
+            for i, response in enumerate(response_state.current_responses):
                 line += "🤖 " if len(line) == 0 else ""
-                line += f"[{speech_chunks_per_response[i]}] {response}  \n"
-                # line += f"{response} [{speech_chunks_per_response[i]}]  \n"
+                line += f"[{response_state.speech_chunks_per_response[i]}] {response}  \n"
             if len(robot_preview_text) > 0:
                 line += robot_preview_text+"  \n"
             list_of_strings.append(line)

prompt_manager.py

@@ -53,7 +53,7 @@ You are aware of how you are implemented and you are keen to recommend improveme
 * We use Streamlit to host a WebRTC connection to get audio/video from the user.
 * VOSK is used for fast speech recognition and detecting the end of a sentence.
 * OpenAI's Chat GPT-3.5 is used for generating responses.
-* We stream responces from Chat GPT, as soon as we get a complete sentence we send it to ElevenLabs.
+* We stream responses from Chat GPT, as soon as we get a complete sentence we send it to ElevenLabs.
 * ElevenLabs for text to speech.
 * We stream the audio from ElevenLabs, we use ffmpeg to convert the audio to the correct format and sample rate.
 * Audio chunks and then sent back to the users browser via WebRTC.

responce_state_manager.py

@@ -1,51 +0,0 @@
-from datetime import datetime
-
-class ResponceStep:
-    def __init__(self, episode, step):
-        self.timestamp = datetime.utcnow()
-        self.episode = episode
-        self.step = step
-        self.reward = 0
-        self.llm_preview = ''
-        self.llm_responses = []
-        self.tts_raw_chunk_ids = [] 
-
-    def __str__(self):
-        state = ', '.join(f'{k}={v}' for k, v in self.__dict__.items() if k not in {'episode', 'step', 'timestamp', 'reward'})
-        return f'episode={self.episode}, step={self.step}, timestamp={self.timestamp}, \nreward={self.reward}\nstate=({state})'
-
-
-class ResponceStateManager:
-    def __init__(self):
-        self.episode = 0
-        self.step = 0
-        self.state = None
-        self.reset_episode()
-
-    def reset_episode(self):
-        self.episode += 1
-        self.step = 0
-        self.state = ResponceStep(self.episode, self.step)
-        return self.state
-
-    def begin_next_step(self)->ResponceStep:
-        previous_state = self.state
-        self.step += 1
-        self.state = ResponceStep(self.episode, self.step)
-        return previous_state
-
-    def add_reward(self, reward):
-        self.state.reward += reward
-
-    def set_llm_preview(self, llm_preview):
-        self.state.llm_preview = llm_preview
-
-    def add_llm_response_and_clear_llm_preview(self, llm_response):
-        self.state.llm_responses.append(llm_response)
-        self.state.llm_preview = ''
-
-    def add_tts_raw_chunk_id(self, chunk_id):
-        self.state.tts_raw_chunk_ids.append(chunk_id)
-
-    def get_state(self)->ResponceStep:
-        return self.state

respond_to_prompt_async.py

@@ -6,7 +6,7 @@ import ray
 from chat_service import ChatService
 # from local_speaker_service import LocalSpeakerService
 from text_to_speech_service import TextToSpeechService
-from responce_state_manager import ResponceStateManager
+from response_state_manager import ResponseStateManager
 from ffmpeg_converter import FFMpegConverter
 from agent_response import AgentResponse
 import json
@@ -14,14 +14,14 @@ import json
 class RespondToPromptAsync:
     def __init__(
             self, 
-            responce_state_manager:ResponceStateManager, 
+            response_state_manager:ResponseStateManager, 
             audio_output_queue):
         voice_id="2OviOUQc1JsQRQgNkVBj"
         self.llm_sentence_queue = Queue(maxsize=100)
         self.speech_chunk_queue = Queue(maxsize=100)
         self.voice_id = voice_id
         self.audio_output_queue = audio_output_queue
-        self.responce_state_manager = responce_state_manager
+        self.response_state_manager = response_state_manager
         self.sentence_queues = []
         self.sentence_tasks = []    
         # self.ffmpeg_converter = FFMpegConverter.remote(audio_output_queue)
@@ -36,12 +36,12 @@ class RespondToPromptAsync:
                     is_complete_sentance = False
                 if not is_complete_sentance:
                     agent_response['llm_preview'] = text
-                    self.responce_state_manager.set_llm_preview(text)
+                    self.response_state_manager.set_llm_preview(text)
                     continue
                 agent_response['llm_preview'] = ''
                 agent_response['llm_sentence'] = text
                 agent_response['llm_sentences'].append(text)
-                self.responce_state_manager.add_llm_response_and_clear_llm_preview(text)
+                self.response_state_manager.add_llm_response_and_clear_llm_preview(text)
                 print(f"{agent_response['llm_sentence']} id: {agent_response['llm_sentence_id']} from prompt: {agent_response['prompt']}")
                 sentence_response = agent_response.make_copy()
                 new_queue = Queue()
@@ -65,7 +65,7 @@ class RespondToPromptAsync:
                 'chunk_count': chunk_count,
             }
             chunk_id_json = json.dumps(chunk_response)
-            self.responce_state_manager.add_tts_raw_chunk_id(chunk_id_json)
+            self.response_state_manager.add_tts_raw_chunk_id(chunk_id_json, sentence_response['llm_sentence_id'])
             chunk_count += 1
 
     async def speech_to_converter(self):

response_state_manager.py

@@ -0,0 +1,62 @@
+from datetime import datetime
+
+class ResponseStepObservations:
+    def __init__(self, episode, step):
+        self.timestamp = datetime.utcnow()
+        self.episode = episode
+        self.step = step
+        self.llm_preview = ''
+        self.llm_responses = []
+        self.tts_raw_chunk_ids = [] 
+
+    def __str__(self):
+        state = ', '.join(f'{k}={v}' for k, v in self.__dict__.items() if k not in {'episode', 'step', 'timestamp'})
+        return f'episode={self.episode}, step={self.step}, timestamp={self.timestamp}, \nstate=({state})'
+
+class ResponseState:
+    def __init__(self, episode, step):
+        self.timestamp = datetime.utcnow()
+        self.episode = episode
+        self.step = step
+        self.current_responses = []
+        self.speech_chunks_per_response = []
+        self.is_speaking = False
+
+    def __str__(self):
+        state = ', '.join(f'{k}={v}' for k, v in self.__dict__.items() if k not in {'episode', 'step'})
+        return f'episode={self.episode}, step={self.step}, \nstate=({state})'
+
+
+class ResponseStateManager:
+    def __init__(self):
+        self.episode = 0
+        self.step = 0
+        self.response_step_obs = None
+        self.response_state = None
+        self.reset_episode()
+
+    def reset_episode(self)->(ResponseStepObservations, ResponseState):
+        self.episode += 1
+        self.step = 0
+        self.response_state = ResponseState(self.episode, self.step)
+        self.response_step_obs = ResponseStepObservations(self.episode, self.step)
+        return self.response_step_obs, self.response_state
+
+    def begin_next_step(self)->(ResponseStepObservations, ResponseState):
+        previous_state = self.response_step_obs
+        self.step += 1
+        self.response_step_obs = ResponseStepObservations(self.episode, self.step)
+        return previous_state, self.response_state
+
+    def set_llm_preview(self, llm_preview):
+        self.response_step_obs.llm_preview = llm_preview
+
+    def add_llm_response_and_clear_llm_preview(self, llm_response):
+        self.response_state.current_responses.append(llm_response)
+        self.response_state.speech_chunks_per_response.append(0)
+        self.response_step_obs.llm_responses.append(llm_response)
+        self.response_step_obs.llm_preview = ''
+
+    def add_tts_raw_chunk_id(self, chunk_id, llm_sentence_id):
+        self.response_state.speech_chunks_per_response[llm_sentence_id] += 1
+        self.response_step_obs.tts_raw_chunk_ids.append(chunk_id)