sohojoe commited on
Commit
795c382
β€’
1 Parent(s): 937fa8f

handle false signal of speaker_finsihed

Browse files
charles_actor.py CHANGED
@@ -100,6 +100,10 @@ class CharlesActor:
100
  speech_chunks_per_response = []
101
  human_preview_text = ""
102
  robot_preview_text = ""
 
 
 
 
103
 
104
 
105
  while True:
@@ -148,7 +152,6 @@ class CharlesActor:
148
  line += f"[{speech_chunks_per_response[i]}] {response} \n"
149
  if len(line) > 0:
150
  add_debug_output(line)
151
- add_debug_output(f"πŸ‘¨ {prompt}")
152
  current_responses = []
153
  speech_chunks_per_response = []
154
  env_state.llm_preview = ""
@@ -156,8 +159,23 @@ class CharlesActor:
156
  env_state.tts_raw_chunk_ids = []
157
  human_preview_text = ""
158
  robot_preview_text = ""
 
 
 
159
  await self._respond_to_prompt_actor.enqueue_prompt.remote(prompt)
 
 
 
 
160
  elif len(prompt) > 0 and prompt not in prompts_to_ignore:
 
 
 
 
 
 
 
 
161
  human_preview_text = f"πŸ‘¨β“ {prompt}"
162
 
163
  for new_response in env_state.llm_responses:
@@ -195,6 +213,7 @@ class CharlesActor:
195
  # add observations to the environment state
196
  count = len(self._out_audio_queue)
197
  is_talking = bool(count > 0)
 
198
  frame = self._animator.update(is_talking)
199
  if self._out_video_queue.full():
200
  evicted_item = await self._out_video_queue.get_async()
 
100
  speech_chunks_per_response = []
101
  human_preview_text = ""
102
  robot_preview_text = ""
103
+ additional_prompt = None
104
+ previous_prompt = ""
105
+ is_talking = False
106
+ has_spoken_for_this_prompt = False
107
 
108
 
109
  while True:
 
152
  line += f"[{speech_chunks_per_response[i]}] {response} \n"
153
  if len(line) > 0:
154
  add_debug_output(line)
 
155
  current_responses = []
156
  speech_chunks_per_response = []
157
  env_state.llm_preview = ""
 
159
  env_state.tts_raw_chunk_ids = []
160
  human_preview_text = ""
161
  robot_preview_text = ""
162
+ if additional_prompt is not None:
163
+ prompt = additional_prompt + ". " + prompt
164
+ add_debug_output(f"πŸ‘¨ {prompt}")
165
  await self._respond_to_prompt_actor.enqueue_prompt.remote(prompt)
166
+ additional_prompt = None
167
+ previous_prompt = prompt
168
+ is_talking = False
169
+ has_spoken_for_this_prompt = False
170
  elif len(prompt) > 0 and prompt not in prompts_to_ignore:
171
+ # sometimes we get a false signal of speaker_finsihed
172
+ # in which case we get new prompts preview before we have spoken
173
+ if len(previous_prompt) > 0 and not has_spoken_for_this_prompt:
174
+ additional_prompt = previous_prompt
175
+ has_spoken_for_this_prompt = True
176
+ await self._respond_to_prompt_actor.enqueue_prompt.remote("")
177
+ if additional_prompt is not None:
178
+ prompt = additional_prompt + ". " + prompt
179
  human_preview_text = f"πŸ‘¨β“ {prompt}"
180
 
181
  for new_response in env_state.llm_responses:
 
213
  # add observations to the environment state
214
  count = len(self._out_audio_queue)
215
  is_talking = bool(count > 0)
216
+ has_spoken_for_this_prompt = has_spoken_for_this_prompt or is_talking
217
  frame = self._animator.update(is_talking)
218
  if self._out_video_queue.full():
219
  evicted_item = await self._out_video_queue.get_async()
respond_to_prompt_actor.py CHANGED
@@ -184,6 +184,7 @@ class RespondToPromptActor:
184
  speech_output_future,
185
  ffmpeg_converter_future,
186
  )
187
- await self.prompt_queue.put_async(prompt)
 
188
  print("Enqueued prompt")
189
 
 
184
  speech_output_future,
185
  ffmpeg_converter_future,
186
  )
187
+ if len(prompt) > 0: # handles case where we just want to flush
188
+ await self.prompt_queue.put_async(prompt)
189
  print("Enqueued prompt")
190
 
webrtc_av_queue_actor.py CHANGED
@@ -59,6 +59,7 @@ class WebRtcAVQueueActor:
59
  async def get_out_video_frame(self):
60
  if self.out_video_queue.empty():
61
  return None
 
62
  while not self.out_video_queue.empty():
63
  frame = await self.out_video_queue.get_async()
64
  return frame
 
59
  async def get_out_video_frame(self):
60
  if self.out_video_queue.empty():
61
  return None
62
+ frame = None
63
  while not self.out_video_queue.empty():
64
  frame = await self.out_video_queue.get_async()
65
  return frame