Create app.py

app.py

@@ -0,0 +1,159 @@
+import os
+# Set CUDA environment variable and install llama-cpp-python
+# llama-cpp-python is a python binding for llama.cpp library which enables LLM inference in pure C/C++
+os.environ["CUDACXX"] = "/usr/local/cuda/bin/nvcc"
+os.system('python -m unidic download')
+os.system('CMAKE_ARGS="-DLLAMA_CUBLAS=on" pip install llama-cpp-python==0.2.11 --verbose')
+
+
+# Third-party library imports
+from faster_whisper import WhisperModel
+import gradio as gr
+from huggingface_hub import hf_hub_download
+from llama_cpp import Llama
+from TTS.tts.configs.xtts_config import XttsConfig
+from TTS.tts.models.xtts import Xtts
+from TTS.utils.generic_utils import get_user_data_dir
+from TTS.utils.manage import ModelManager
+
+# Local imports
+from utils import get_sentence, generate_speech_for_sentence, wave_header_chunk
+
+# Load Whisper ASR model
+print("Loading Whisper ASR")
+whisper_model = WhisperModel("large-v3", device="cuda", compute_type="float16")
+
+# Load Mistral LLM
+print("Loading Mistral LLM")
+hf_hub_download(repo_id="TheBloke/Mistral-7B-Instruct-v0.1-GGUF", local_dir=".", filename="mistral-7b-instruct-v0.1.Q5_K_M.gguf")
+mistral_model_path="./mistral-7b-instruct-v0.1.Q5_K_M.gguf"
+mistral_llm = Llama(model_path=mistral_model_path,n_gpu_layers=35,max_new_tokens=256, context_window=4096, n_ctx=4096,n_batch=128,verbose=False)
+
+
+# Load XTTS Model
+print("Loading XTTS model")
+os.environ["COQUI_TOS_AGREED"] = "1"
+tts_model_name = "tts_models/multilingual/multi-dataset/xtts_v2"
+ModelManager().download_model(tts_model_name)
+tts_model_path = os.path.join(get_user_data_dir("tts"), tts_model_name.replace("/", "--"))
+config = XttsConfig()
+config.load_json(os.path.join(tts_model_path, "config.json"))
+xtts_model = Xtts.init_from_config(config)
+xtts_model.load_checkpoint(
+    config,
+    checkpoint_path=os.path.join(tts_model_path, "model.pth"),
+    vocab_path=os.path.join(tts_model_path, "vocab.json"),
+    eval=True,
+    use_deepspeed=True,
+)
+xtts_model.cuda()
+
+###### Set up Gradio Interface ######
+
+with gr.Blocks(title="Voice chat with LLM") as demo:
+    DESCRIPTION = """# Voice chat with LLM"""
+    gr.Markdown(DESCRIPTION)
+
+    # Define chatbot component
+    chatbot = gr.Chatbot(
+        value=[(None, "Hi friend, I'm Liya Interviewer and I'm here to help you with that today?")],  # Initial greeting from the chatbot
+        elem_id="chatbot",
+        avatar_images=("examples/hf-logo.png", "examples/ai-chat-logo.png"),
+        bubble_full_width=False,
+    )
+
+    # Define chatbot voice component
+    VOICES = ["female", "male"]
+    with gr.Row():
+        chatbot_voice = gr.Dropdown(
+            label="Voice of the Chatbot",
+            info="How should Chatbot talk like",
+            choices=VOICES,
+            max_choices=1,
+            value=VOICES[0],
+        )
+
+    # Define text and audio record input components
+    with gr.Row():
+        txt_box = gr.Textbox(
+            scale=3,
+            show_label=False,
+            placeholder="Enter text and press enter, or speak to your microphone",
+            container=False,
+            interactive=True,
+        )
+        audio_record = gr.Audio(source="microphone", type="filepath", scale=4)
+
+    # Define generated audio playback component 
+    with gr.Row():
+        sentence = gr.Textbox(visible=False)
+        audio_playback = gr.Audio(
+            value=None,
+            label="Generated audio response",
+            streaming=True,
+            autoplay=True,
+            interactive=False,
+            show_label=True,
+        )
+
+    # Will be triggered on text submit (will send to generate_speech)
+    def add_text(chatbot_history, text):
+        chatbot_history = [] if chatbot_history is None else chatbot_history
+        chatbot_history = chatbot_history + [(text, None)]
+        return chatbot_history, gr.update(value="", interactive=False)
+    
+    # Will be triggered on voice submit (will transribe and send to generate_speech)
+    def add_audio(chatbot_history, audio):
+        chatbot_history = [] if chatbot_history is None else chatbot_history
+        # get result from whisper and strip it to delete begin and end space
+        response, _ = whisper_model.transcribe(audio)
+        text = list(response)[0].text.strip()
+        print("Transcribed text:", text)
+        chatbot_history = chatbot_history + [(text, None)]
+        return chatbot_history, gr.update(value="", interactive=False)
+
+    def generate_speech(chatbot_history, chatbot_voice, initial_greeting=False):
+        # Start by yielding an initial empty audio to set up autoplay
+        yield ("", chatbot_history, wave_header_chunk())
+
+        # Helper function to handle the speech generation and yielding process
+        def handle_speech_generation(sentence, chatbot_history, chatbot_voice):
+            if sentence != "":
+                print("Processing sentence")
+                generated_speech = generate_speech_for_sentence(chatbot_history, chatbot_voice, sentence, xtts_model, xtts_supported_languages=config.languages, return_as_byte=True)
+                if generated_speech is not None:
+                    _, audio_dict = generated_speech
+                    yield (sentence, chatbot_history, audio_dict["value"])
+
+        if initial_greeting:
+            # Process only the initial greeting if specified
+            for _, sentence in chatbot_history:
+                yield from handle_speech_generation(sentence, chatbot_history, chatbot_voice)
+        else:
+            # Continuously get and process sentences from a generator function
+            for sentence, chatbot_history in get_sentence(chatbot_history, mistral_llm):
+                print("Inserting sentence to queue")
+                yield from handle_speech_generation(sentence, chatbot_history, chatbot_voice)
+
+    txt_msg = txt_box.submit(fn=add_text, inputs=[chatbot, txt_box], outputs=[chatbot, txt_box], queue=False
+                             ).then(fn=generate_speech,  inputs=[chatbot,chatbot_voice], outputs=[sentence, chatbot, audio_playback])
+
+    txt_msg.then(fn=lambda: gr.update(interactive=True), inputs=None, outputs=[txt_box], queue=False)
+
+    audio_msg = audio_record.stop_recording(fn=add_audio, inputs=[chatbot, audio_record], outputs=[chatbot, txt_box], queue=False
+                                            ).then(fn=generate_speech,  inputs=[chatbot,chatbot_voice], outputs=[sentence, chatbot, audio_playback])
+
+    audio_msg.then(fn=lambda: (gr.update(interactive=True),gr.update(interactive=True,value=None)), inputs=None, outputs=[txt_box, audio_record], queue=False)
+
+    FOOTNOTE = """
+            This Space demonstrates how to speak to an llm chatbot, based solely on open accessible models.
+            It relies on the following models :
+            - Speech to Text Model: [Faster-Whisper-large-v3](https://huggingface.co/Systran/faster-whisper-large-v3) an ASR model, to transcribe recorded audio to text.
+            - Large Language Model: [Mistral-7b-instruct-v0.1-quantized](https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.1-GGUF) a LLM to generate the chatbot responses. 
+            - Text to Speech Model: [XTTS-v2](https://huggingface.co/spaces/coqui/xtts) a TTS model, to generate the voice of the chatbot.
+            Note:
+            - Responses generated by chat model should not be assumed correct or taken serious, as this is a demonstration example only
+            - iOS (Iphone/Ipad) devices may not experience voice due to autoplay being disabled on these devices by Vendor"""
+    gr.Markdown(FOOTNOTE)
+    demo.load(fn=generate_speech, inputs=[chatbot,chatbot_voice, gr.State(value=True)], outputs=[sentence, chatbot, audio_playback])
+demo.queue().launch(debug=True,share=True)