Duplicate from Yusin/Speech-ChatGPT-Speech

Co-authored-by: Chen, Yusin <Yusin@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,27 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+.env

README.md

@@ -0,0 +1,41 @@
+---
+title: Speech2ChatGPT2Speech
+emoji: 🗣️🙉
+colorFrom: indigo
+colorTo: yellow
+sdk: gradio
+python_version: 3.9
+sdk_version: 3.12.0
+app_file: app.py
+models:
+- neongeckocom/tts-vits-ljspeech-en
+- neongeckocom/tts-vits-css10-es
+- neongeckocom/tts-vits-css10-fr
+- neongeckocom/tts-vits-css10-de
+- neongeckocom/tts-vits-cv-it
+- neongeckocom/tts-vits-mai-pl
+- neongeckocom/tts-vits-mai-uk
+- neongeckocom/tts-vits-cv-ro
+- neongeckocom/tts-vits-css10-hu
+- neongeckocom/tts-vits-cv-el
+- neongeckocom/tts-vits-cv-cs
+- neongeckocom/tts-vits-cv-sv
+- neongeckocom/tts-vits-cv-pt
+- neongeckocom/tts-vits-cv-bg
+- neongeckocom/tts-vits-cv-hr
+- neongeckocom/tts-vits-cv-da
+- neongeckocom/tts-vits-cv-sk
+- neongeckocom/tts-vits-css10-nl
+- neongeckocom/tts-vits-css10-fi
+- neongeckocom/tts-vits-cv-lt
+- neongeckocom/tts-vits-cv-sl
+- neongeckocom/tts-vits-cv-lv
+- neongeckocom/tts-vits-cv-et
+- neongeckocom/tts-vits-cv-ga
+- neongeckocom/tts-vits-cv-mt
+pinned: false
+license: apache-2.0
+duplicated_from: Yusin/Speech-ChatGPT-Speech
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces#reference

app.py

@@ -0,0 +1,154 @@
+import tempfile
+import gradio as gr
+from neon_tts_plugin_coqui import CoquiTTS
+LANGUAGES = list(CoquiTTS.langs.keys())
+default_lang = "en"
+import telnetlib
+#import whisper
+#whisper_model = whisper.load_model("small")
+whisper = gr.Interface.load(name="spaces/sanchit-gandhi/whisper-large-v2")
+#chatgpt = gr.Blocks.load(name="spaces/fffiloni/whisper-to-chatGPT")
+import os
+import json
+session_token = os.environ.get('SessionToken')
+#api_endpoint = os.environ.get('API_EndPoint')
+# ChatGPT
+#from revChatGPT.ChatGPT import Chatbot
+#chatbot = Chatbot({"session_token": session_token}) # You can start a custom conversation
+import asyncio
+from pygpt import PyGPT
+
+title = "Speech to ChatGPT to Speech"
+#info = "more info at [Neon Coqui TTS Plugin](https://github.com/NeonGeckoCom/neon-tts-plugin-coqui), [Coqui TTS](https://github.com/coqui-ai/TTS)"
+#badge = "https://visitor-badge-reloaded.herokuapp.com/badge?page_id=neongeckocom.neon-tts-plugin-coqui"
+coquiTTS = CoquiTTS()
+chat_id = {'conversation_id': None, 'parent_id': None}
+headers = {'Authorization': 'yusin'}
+
+async def chat_gpt_ask(prompt):
+    chat_gpt = PyGPT(session_token)
+    await chat_gpt.connect()
+    await chat_gpt.wait_for_ready()
+    answer = await chat_gpt.ask(prompt)
+    print(answer)
+    await chat_gpt.disconnect()
+
+# ChatGPT
+def chat_hf(audio, custom_token, language):
+    #output = chatgpt(audio, "transcribe", fn_index=0)
+    #whisper_text, gpt_response = output[0], output[1]
+    try:
+        whisper_text = translate(audio)
+        if whisper_text == "ERROR: You have to either use the microphone or upload an audio file":
+            gpt_response = "MISSING AUDIO: Record your voice by clicking the microphone button, do not forget to stop recording before sending your message ;)"
+        else:
+            #gpt_response = chatbot.ask(whisper_text, conversation_id=conversation_id, parent_id=None)
+            gpt_response = asyncio.run(chat_gpt_ask(whisper_text, id='yusin'))
+            #if chat_id['conversation_id'] != None:
+            #    data = {"content": whisper_text, "conversation_id": chat_id['conversation_id'], "parent_id": chat_id['parent_id']}
+            #else: 
+            #    data = {"content": whisper_text}
+            #print(data)
+            #res = requests.get('http://myip.ipip.net', timeout=5).text
+            #print(res)
+            #response = requests.post('api_endpoint', headers=headers, json=data, verify=False, timeout=5)
+            #print('this is my answear', response.text)
+            #chat_id['parent_id'] = response.json()["response_id"]
+            #chat_id['conversation_id'] = response.json()["conversation_id"]
+            #gpt_response = response.json()["content"]
+            #response = requests.get('https://api.pawan.krd/chat/gpt?text=' + whisper_text + '&cache=false', verify=False, timeout=5)
+            #print(response.text)
+        
+        #whisper_text = translate(audio)
+        #api = ChatGPT(session_token) 
+        #resp = api.send_message(whisper_text)
+        
+        #api.refresh_auth()  # refresh the authorization token
+        #api.reset_conversation()  # reset the conversation
+        #gpt_response = resp['message']
+
+    except:
+        whisper_text = translate(audio)
+        gpt_response = """Sorry, I'm quite busy right now, but please try again later :)"""
+        #whisper_text = translate(audio)
+        #api = ChatGPT(custom_token) 
+        #resp = api.send_message(whisper_text)
+        
+        #api.refresh_auth()  # refresh the authorization token
+        #api.reset_conversation()  # reset the conversation
+        #gpt_response = resp['message']
+    
+    # to voice
+    with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as fp:
+        coquiTTS.get_tts(gpt_response, fp, speaker = {"language" : language})
+    
+    return whisper_text, gpt_response, fp.name
+
+# whisper
+#def translate(audio):
+#    print("""
+#    —
+#    Sending audio to Whisper ...
+#    —
+#    """)
+#    
+#    audio = whisper.load_audio(audio)
+#    audio = whisper.pad_or_trim(audio)
+#    
+#    mel = whisper.log_mel_spectrogram(audio).to(whisper_model.device)
+#    
+#    _, probs = whisper_model.detect_language(mel)
+#    
+#    transcript_options = whisper.DecodingOptions(task="transcribe", fp16 = False)
+#    
+#    transcription = whisper.decode(whisper_model, mel, transcript_options)
+#    
+#    print("language spoken: " + transcription.language)
+#    print("transcript: " + transcription.text)
+#    print("———————————————————————————————————————————")  
+#      
+#    return transcription.text
+
+def translate(audio):
+    print("""
+    —
+    Sending audio to Whisper ...
+    —
+    """)
+   
+    text_result = whisper(audio, None, "transcribe", fn_index=0)
+    #print(text_result)
+    return text_result
+
+
+with gr.Blocks() as blocks:
+    gr.Markdown("<h1 style='text-align: center; margin-bottom: 1rem'>"
+                + title
+                + "</h1>")
+    #gr.Markdown(description)
+    radio = gr.Radio(label="Language",choices=LANGUAGES,value=default_lang)
+    with gr.Row(equal_height=True):# equal_height=False
+        with gr.Column():# variant="panel"
+            audio_file = gr.Audio(source="microphone",type="filepath")
+            custom_token = gr.Textbox(label='If it fails, use your own session token', placeholder="your own session token")
+            with gr.Row():# mobile_collapse=False
+                submit = gr.Button("Submit", variant="primary")
+        with gr.Column():
+            text1 = gr.Textbox(label="Speech to Text")
+            text2 = gr.Textbox(label="ChatGPT Response")
+            audio = gr.Audio(label="Output", interactive=False)
+    #gr.Markdown(info)
+    #gr.Markdown("<center>"
+    #            +f'<img src={badge} alt="visitors badge"/>'
+    #            +"</center>")
+
+    # actions
+    submit.click(
+        chat_hf,
+        [audio_file, custom_token, radio],
+        [text1, text2, audio],
+    )
+    radio.change(lambda lang: CoquiTTS.langs[lang]["sentence"], radio, text2)
+
+
+blocks.launch(debug=True)

packages.txt

@@ -0,0 +1,2 @@
+libsndfile1
+espeak-ng

pygpt.py

@@ -0,0 +1,111 @@
+import uuid
+import asyncio
+import socketio
+import datetime
+import json
+import base64
+
+class PyGPT:
+    def __init__(self, session_token, bypass_node='https://gpt.pawan.krd'):
+        self.ready = False
+        self.socket = socketio.AsyncClient()
+        self.socket.on('connect', self.on_connect)
+        self.socket.on('disconnect', self.on_disconnect)
+        self.session_token = session_token
+        self.conversations = []
+        self.auth = None
+        self.expires = datetime.datetime.now()
+        self.pause_token_checks = False
+        self.bypass_node = bypass_node
+        asyncio.create_task(self.cleanup_conversations())
+
+    async def connect(self):
+        await self.socket.connect(self.bypass_node)
+
+    async def disconnect(self):
+        await self.socket.disconnect()
+
+    def on_connect(self):
+        print('Connected to server')
+        asyncio.create_task(self.check_tokens())
+
+    def on_disconnect(self):
+        print('Disconnected from server')
+        self.ready = False
+
+    async def check_tokens(self):
+        while True:
+            if self.pause_token_checks:
+                await asyncio.sleep(0.5)
+                continue
+            self.pause_token_checks = True
+            now = datetime.datetime.now()
+            offset = datetime.timedelta(minutes=2)
+            if self.expires < (now - offset) or not self.auth:
+                await self.get_tokens()
+            self.pause_token_checks = False
+            await asyncio.sleep(0.5)
+
+    async def cleanup_conversations(self):
+        while True:
+            await asyncio.sleep(60)
+            now = datetime.datetime.now()
+            self.conversations = [c for c in self.conversations if now - c['last_active'] < datetime.timedelta(minutes=2)]
+
+    def add_conversation(self, id):
+        conversation = {
+            'id': id,
+            'conversation_id': None,
+            'parent_id': uuid.uuid4(),
+            'last_active': datetime.datetime.now()
+        }
+        self.conversations.append(conversation)
+        return conversation
+
+    def get_conversation_by_id(self, id):
+        conversation = next((c for c in self.conversations if c['id'] == id), None)
+        if conversation is None:
+            conversation = self.add_conversation(id)
+        else:
+            conversation['last_active'] = datetime.datetime.now()
+        return conversation
+
+    async def wait_for_ready(self):
+        while not self.ready:
+            await asyncio.sleep(0.025)
+        print('Ready!!')
+
+    async def ask(self, prompt, id='default'):
+        if not self.auth or not self.validate_token(self.auth):
+            await self.get_tokens()
+        conversation = self.get_conversation_by_id(id)
+        data = await self.socket.call('askQuestion', {
+            'prompt': prompt,
+            'parentId': str(conversation['parent_id']),
+            'conversationId': str(conversation['conversation_id']),
+            'auth': self.auth
+        })
+
+        if 'error' in data:
+            print(f'Error: {data["error"]}')
+        conversation['parent_id'] = data['messageId']
+        conversation['conversation_id'] = data['conversationId']
+        return data['answer']
+
+    def validate_token(self, token):
+        if not token:
+            return False
+        parsed = json.loads(base64.b64decode(f'{token.split(".")[1]}==').decode())
+        return datetime.datetime.now() <= datetime.datetime.fromtimestamp(parsed['exp'])
+
+    async def get_tokens(self):
+        await asyncio.sleep(1)
+        data = await self.socket.call('getSession', self.session_token)
+
+        if 'error' in data:
+            print(f'Error getting session: {data["error"]}')
+        else:
+            self.auth = data['auth']
+            self.expires = datetime.datetime.strptime(data['expires'], '%Y-%m-%dT%H:%M:%S.%fZ')
+            self.session_token = data['sessionToken']
+            self.ready = True

requirements.txt

@@ -0,0 +1,5 @@
+neon-tts-plugin-coqui==0.7.0
+python-socketio[asyncio_client]
+#undetected-chromedriver
+#revChatGPT
+#git+https://github.com/openai/whisper.git