Update vehicle destination in calculate_route function

core/__init__.py

@@ -0,0 +1,176 @@
+import os
+from collections import namedtuple
+import time
+import pathlib
+from typing import List
+
+import numpy as np
+import torch
+from TTS.api import TTS
+
+os.environ["COQUI_TOS_AGREED"] = "1"
+
+
+Voice = namedtuple("voice", ["name", "neutral", "angry"])
+
+file_full_path = pathlib.Path(os.path.realpath(__file__)).parent
+
+voices = [
+    Voice(
+        "Attenborough",
+        neutral=f"{file_full_path}/audio/attenborough/neutral.wav",
+        angry=None,
+    ),
+    Voice("Rick", neutral=f"{file_full_path}/audio/rick/neutral.wav", angry=None),
+    Voice(
+        "Freeman",
+        neutral=f"{file_full_path}/audio/freeman/neutral.wav",
+        angry="audio/freeman/angry.wav",
+    ),
+    Voice("Walken", neutral=f"{file_full_path}/audio/walken/neutral.wav", angry=None),
+    Voice(
+        "Darth Wader", neutral=f"{file_full_path}/audio/darth/neutral.wav", angry=None
+    ),
+]
+
+
+def load_tts_pipeline():
+    # load model for text to speech
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    # device = "mps"
+    tts_pipeline = TTS("tts_models/multilingual/multi-dataset/xtts_v2").to(device)
+    return tts_pipeline
+
+
+def compute_speaker_embedding(voice_path: str, config, pipeline, cache):
+    if voice_path not in cache:
+        cache[voice_path] = pipeline.synthesizer.tts_model.get_conditioning_latents(
+            audio_path=voice_path,
+            gpt_cond_len=config.gpt_cond_len,
+            gpt_cond_chunk_len=config.gpt_cond_chunk_len,
+            max_ref_length=config.max_ref_len,
+            sound_norm_refs=config.sound_norm_refs,
+        )
+    return cache[voice_path]
+
+
+voice_options = []
+for voice in voices:
+    if voice.neutral:
+        voice_options.append(f"{voice.name} - Neutral")
+    if voice.angry:
+        voice_options.append(f"{voice.name} - Angry")
+
+
+def voice_from_text(voice):
+    for v in voices:
+        if voice == f"{v.name} - Neutral":
+            return v.neutral
+        if voice == f"{v.name} - Angry":
+            return v.angry
+    raise ValueError(f"Voice {voice} not found.")
+
+
+def tts(
+    self,
+    text: str = "",
+    language_name: str = "",
+    reference_wav=None,
+    gpt_cond_latent=None,
+    speaker_embedding=None,
+    split_sentences: bool = True,
+    **kwargs,
+) -> List[int]:
+    """🐸 TTS magic. Run all the models and generate speech.
+
+    Args:
+        text (str): input text.
+        speaker_name (str, optional): speaker id for multi-speaker models. Defaults to "".
+        language_name (str, optional): language id for multi-language models. Defaults to "".
+        speaker_wav (Union[str, List[str]], optional): path to the speaker wav for voice cloning. Defaults to None.
+        style_wav ([type], optional): style waveform for GST. Defaults to None.
+        style_text ([type], optional): transcription of style_wav for Capacitron. Defaults to None.
+        reference_wav ([type], optional): reference waveform for voice conversion. Defaults to None.
+        reference_speaker_name ([type], optional): speaker id of reference waveform. Defaults to None.
+        split_sentences (bool, optional): split the input text into sentences. Defaults to True.
+        **kwargs: additional arguments to pass to the TTS model.
+    Returns:
+        List[int]: [description]
+    """
+    start_time = time.time()
+    use_gl = self.vocoder_model is None
+    wavs = []
+
+    if not text and not reference_wav:
+        raise ValueError(
+            "You need to define either `text` (for sythesis) or a `reference_wav` (for voice conversion) to use the Coqui TTS API."
+        )
+
+    if text:
+        sens = [text]
+        if split_sentences:
+            print(" > Text splitted to sentences.")
+            sens = self.split_into_sentences(text)
+        print(sens)
+
+    if not reference_wav:  # not voice conversion
+        for sen in sens:
+            outputs = self.tts_model.inference(
+                sen,
+                language_name,
+                gpt_cond_latent,
+                speaker_embedding,
+                # GPT inference
+                temperature=0.75,
+                length_penalty=1.0,
+                repetition_penalty=10.0,
+                top_k=50,
+                top_p=0.85,
+                do_sample=True,
+                **kwargs,
+            )
+            waveform = outputs["wav"]
+            if (
+                torch.is_tensor(waveform)
+                and waveform.device != torch.device("cpu")
+                and not use_gl
+            ):
+                waveform = waveform.cpu()
+            if not use_gl:
+                waveform = waveform.numpy()
+            waveform = waveform.squeeze()
+
+            # # trim silence
+            # if (
+            #     "do_trim_silence" in self.tts_config.audio
+            #     and self.tts_config.audio["do_trim_silence"]
+            # ):
+            #     waveform = trim_silence(waveform, self.tts_model.ap)
+
+            wavs += list(waveform)
+            wavs += [0] * 10000
+
+    # compute stats
+    process_time = time.time() - start_time
+    audio_time = len(wavs) / self.tts_config.audio["sample_rate"]
+    print(f" > Processing time: {process_time}")
+    print(f" > Real-time factor: {process_time / audio_time}")
+    return wavs
+
+
+def tts_gradio(tts_pipeline, text, voice, cache):
+    voice_path = voice_from_text(voice)
+    (gpt_cond_latent, speaker_embedding) = compute_speaker_embedding(
+        voice_path, tts_pipeline.synthesizer.tts_config, tts_pipeline, cache
+    )
+    out = tts(
+        tts_pipeline.synthesizer,
+        text,
+        language_name="en",
+        speaker=None,
+        gpt_cond_latent=gpt_cond_latent,
+        speaker_embedding=speaker_embedding,
+        speed=1.1,
+        # file_path="out.wav",
+    )
+    return (22050, np.array(out)), dict(text=text, voice=voice)

kitt.py

@@ -1,5 +1,12 @@
+import time
 import gradio as gr
+import numpy as np
 import requests
+import torch
+import torchaudio
+from transformers import pipeline
+
+
 
 import skills
 from skills.common import config, vehicle
@@ -21,6 +28,7 @@ from skills import (
     date_time_info
 )
 from skills import extract_func_args
+from core import voice_options, load_tts_pipeline, tts_gradio
 
 
 global_context = {
@@ -29,6 +37,7 @@ global_context = {
     "route_points": [],
 }
 
+speaker_embedding_cache = {}
 
 MODEL_FUNC = "nexusraven"
 MODEL_GENERAL = "llama3:instruct"
@@ -102,7 +111,7 @@ def run_generic_model(query):
     return out["response"]
 
 
-def run_model(query):
+def run_model(query, voice_character):
     print("Query: ", query)
     global_context["query"] = query
     global_context["prompt"] = get_prompt(RAVEN_PROMPT_FUNC, query, "", tools)
@@ -124,11 +133,60 @@ def run_model(query):
         func_name, kwargs = extract_func_args(llm_response)
         print(f"Function: {func_name}, Args: {kwargs}")
         if func_name == "do_anything_else":
-            return run_generic_model(query)
-    
-        return use_tool(func_name, kwargs, tools)
-    return out["response"]
+            output_text = run_generic_model(query)
+        else:
+            output_text = use_tool(func_name, kwargs, tools)
+    else:
+        output_text = out["response"]
+
+    if type(output_text) == tuple:
+        output_text = output_text[0]
+    gr.Info(f"Output text: {output_text}, generating voice output...")
+    return output_text, tts_gradio(tts_pipeline, output_text, voice_character, speaker_embedding_cache)[0]
+
+
+def calculate_route_gradio(origin, destination):
+    plot, vehicle_status, points = calculate_route(origin, destination)
+    global_context["route_points"] = points
+    vehicle.location_coordinates = points[0]["latitude"], points[0]["longitude"]
+    return plot, vehicle_status
+
+
+def update_vehicle_status(trip_progress):
+    n_points = len(global_context["route_points"])
+    new_coords = global_context["route_points"][max(int(trip_progress / 100 * n_points), n_points - 1)]
+    new_coords = new_coords["latitude"], new_coords["longitude"]
+    print(f"Trip progress: {trip_progress}, len: {n_points}, new_coords: {new_coords}")
+    vehicle.location_coordinates = new_coords
+    return vehicle.model_dump_json()
+
 
+device = "cuda" if torch.cuda.is_available() else "cpu"
+transcriber = pipeline("automatic-speech-recognition", model="openai/whisper-base.en", device=device)
+
+
+def save_audio_as_wav(data, sample_rate, file_path):
+    # make a tensor from the numpy array
+    data = torch.tensor(data).reshape(1, -1)
+    torchaudio.save(file_path, data, sample_rate=sample_rate, bits_per_sample=16, encoding="PCM_S")
+
+
+def save_and_transcribe_audio(audio):
+    # capture the audio and save it to a file as wav or mp3
+    # file_name = save("audioinput.wav")
+    sr, y = audio
+    # y = y.astype(np.float32)
+    # y /= np.max(np.abs(y))
+
+    # add timestamp to file name
+    filename = f"recordings/audio{time.time()}.wav"
+    save_audio_as_wav(y, sr, filename)
+    
+    sr, y = audio
+    y = y.astype(np.float32)
+    y /= np.max(np.abs(y))
+    text = transcriber({"sampling_rate": sr, "raw":y})["text"]
+    return text
 
 # to be able to use the microphone on chrome, you will have to go to chrome://flags/#unsafely-treat-insecure-origin-as-secure and enter http://10.186.115.21:7860/
 # in "Insecure origins treated as secure", enable it and relaunch chrome
@@ -138,6 +196,9 @@ def run_model(query):
 # What's the closest restaurant from here?
 
 
+tts_pipeline = load_tts_pipeline()
+
+
 with gr.Blocks(theme=gr.themes.Default()) as demo:
     state = gr.State(
         value={
@@ -146,6 +207,7 @@ with gr.Blocks(theme=gr.themes.Default()) as demo:
             "route_points": [],
         }
     )
+    trip_points = gr.State(value=[])
 
     with gr.Row():
         with gr.Column(scale=1, min_width=300):
@@ -161,24 +223,7 @@ with gr.Blocks(theme=gr.themes.Default()) as demo:
                 value="No",
                 interactive=True,
             )
-            voice_character = gr.Radio(
-                choices=[
-                    "Morgan Freeman",
-                    "Eddie Murphy",
-                    "David Attenborough",
-                    "Rick Sanches",
-                ],
-                label="Choose a voice",
-                value="Morgan Freeman",
-                show_label=True,
-                interactive=True,
-            )
-            emotion = gr.Radio(
-                choices=["Cheerful", "Grumpy"],
-                label="Choose an emotion",
-                value="Cheerful",
-                show_label=True,
-            )
+            voice_character = gr.Radio(choices=voice_options, label='Choose a voice', value=voice_options[0], show_label=True)
             origin = gr.Textbox(
                 value="Luxembourg Gare, Luxembourg", label="Origin", interactive=True
             )
@@ -190,13 +235,14 @@ with gr.Blocks(theme=gr.themes.Default()) as demo:
 
         with gr.Column(scale=2, min_width=600):
             map_plot = gr.Plot()
+            trip_progress = gr.Slider(0, 100, step=5, label="Trip progress", interactive=True)
 
             # map_if = gr.Interface(fn=plot_map, inputs=year_input, outputs=map_plot)
 
     with gr.Row():
         with gr.Column():
-            recorder = gr.Audio(
-                type="filepath", label="Input audio", elem_id="recorder"
+            input_audio = gr.Audio(
+                type="numpy",sources=["microphone"], label="Input audio", elem_id="input_audio"
             )
             input_text = gr.Textbox(
                 value="How is the weather?", label="Input text", interactive=True
@@ -205,7 +251,7 @@ with gr.Blocks(theme=gr.themes.Default()) as demo:
                 value=vehicle.model_dump_json(), label="Vehicle status"
             )
         with gr.Column():
-            output_audio = gr.Audio(label="output audio")
+            output_audio = gr.Audio(label="output audio", autoplay=True)
             output_text = gr.TextArea(value="", label="Output text", interactive=False)
     # iface = gr.Interface(
     #     fn=transcript,
@@ -226,12 +272,12 @@ with gr.Blocks(theme=gr.themes.Default()) as demo:
     # Update plot based on the origin and destination
     # Sets the current location and destination
     origin.submit(
-        fn=calculate_route,
+        fn=calculate_route_gradio,
         inputs=[origin, destination],
         outputs=[map_plot, vehicle_status],
     )
     destination.submit(
-        fn=calculate_route,
+        fn=calculate_route_gradio,
         inputs=[origin, destination],
         outputs=[map_plot, vehicle_status],
     )
@@ -240,7 +286,17 @@ with gr.Blocks(theme=gr.themes.Default()) as demo:
     time_picker.select(fn=set_time, inputs=[time_picker], outputs=[vehicle_status])
 
     # Run the model if the input text is changed
-    input_text.submit(fn=run_model, inputs=[input_text], outputs=[output_text])
+    input_text.submit(fn=run_model, inputs=[input_text, voice_character], outputs=[output_text, output_audio])
+
+    # Set the vehicle status based on the trip progress
+    trip_progress.release(
+        fn=update_vehicle_status, inputs=[trip_progress], outputs=[vehicle_status]
+    )
+
+    # Save and transcribe the audio
+    input_audio.stop_recording(
+        fn=save_and_transcribe_audio, inputs=[input_audio], outputs=[input_text]
+    )
 
 # close all interfaces open to make the port available
 gr.close_all()

skills/poi.py

@@ -86,7 +86,7 @@ def find_points_of_interest(lat="0", lon="0", type_of_poi="restaurant"):
 
     # Format and limit to top 5 results
     formatted_results = [
-        f"The {type_of_poi} {result['poi']['name']} is {int(result['dist'])} meters away"
+        f"The {type_of_poi} {result['poi']['name']}, {int(result['dist'])} meters away"
         for result in sorted_results[:5]
     ]
 

skills/routing.py

@@ -64,7 +64,7 @@ def calculate_route(origin, destination):
     data = response.json()
     points = data["routes"][0]["legs"][0]["points"]
 
-    return plot_route(points), vehicle.model_dump_json()
+    return plot_route(points), vehicle.model_dump_json(), points
 
 
 def find_route_tomtom(

skills/weather.py

@@ -39,12 +39,12 @@ def get_weather(location:str= ""):
     humidity = weather_data['current']['humidity']
     feelslike_c = weather_data['current']['feelslike_c']
 
-    # Formulate the sentences
+    # Formulate the sentences - {region}, {country}
     weather_sentences = (
-        f"The current weather in {location}, {region}, {country} is {condition_text} "
+        f"The current weather in {location} is {condition_text} "
         f"with a temperature of {temperature_c}°C that feels like {feelslike_c}°C. "
-        f"Humidity is at {humidity}%. "
-        f"Wind speed is {wind_kph} kph." if 'wind_kph' in weather_data['current'] else ""
+        # f"Humidity is at {humidity}%. "
+        # f"Wind speed is {wind_kph} kph." if 'wind_kph' in weather_data['current'] else ""
     )
     return weather_sentences, weather_data