Duplicate from Yusin/Speech-ChatGPT-Speech

.gitattributes

@@ -0,0 +1,27 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+.env

README.md

@@ -0,0 +1,41 @@
+---
+title: Speech2ChatGPT2Speech
+emoji: 🗣️🙉
+colorFrom: indigo
+colorTo: yellow
+sdk: gradio
+python_version: 3.9
+sdk_version: 3.12.0
+app_file: app.py
+models:
+- neongeckocom/tts-vits-ljspeech-en
+- neongeckocom/tts-vits-css10-es
+- neongeckocom/tts-vits-css10-fr
+- neongeckocom/tts-vits-css10-de
+- neongeckocom/tts-vits-cv-it
+- neongeckocom/tts-vits-mai-pl
+- neongeckocom/tts-vits-mai-uk
+- neongeckocom/tts-vits-cv-ro
+- neongeckocom/tts-vits-css10-hu
+- neongeckocom/tts-vits-cv-el
+- neongeckocom/tts-vits-cv-cs
+- neongeckocom/tts-vits-cv-sv
+- neongeckocom/tts-vits-cv-pt
+- neongeckocom/tts-vits-cv-bg
+- neongeckocom/tts-vits-cv-hr
+- neongeckocom/tts-vits-cv-da
+- neongeckocom/tts-vits-cv-sk
+- neongeckocom/tts-vits-css10-nl
+- neongeckocom/tts-vits-css10-fi
+- neongeckocom/tts-vits-cv-lt
+- neongeckocom/tts-vits-cv-sl
+- neongeckocom/tts-vits-cv-lv
+- neongeckocom/tts-vits-cv-et
+- neongeckocom/tts-vits-cv-ga
+- neongeckocom/tts-vits-cv-mt
+pinned: false
+license: apache-2.0
+duplicated_from: Yusin/Speech-ChatGPT-Speech
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces#reference

app.py

@@ -0,0 +1,132 @@
+import tempfile
+import gradio as gr
+from neon_tts_plugin_coqui import CoquiTTS
+LANGUAGES = list(CoquiTTS.langs.keys())
+LANGUAGES = LANGUAGES + ['cn', 'jp']
+default_lang = "en"
+#import whisper
+#whisper_model = whisper.load_model("small")
+#whisper = gr.Interface.load(name="spaces/abidlabs/whisper-large-v2")
+whisper = gr.Interface.load(name="spaces/sanchit-gandhi/whisper-large-v2")
+#cn_a_jp = gr.Blocks.load(name="spaces/Yusin/anime-tts_yusin")
+#chatgpt = gr.Blocks.load(name="spaces/fffiloni/whisper-to-chatGPT")
+#chatgpt = gr.Blocks.load(name="spaces/seawolf2357/chatgptclone")
+import os
+import json
+import openai
+#session_token = os.environ.get('SessionToken')
+api_key = os.environ.get('api_key')
+#if you have OpenAI API key as a string, enable the below
+openai.api_key = api_key
+
+title = "Speech to ChatGPT to Speech"
+#info = "more info at [Neon Coqui TTS Plugin](https://github.com/NeonGeckoCom/neon-tts-plugin-coqui), [Coqui TTS](https://github.com/coqui-ai/TTS)"
+#badge = "https://visitor-badge-reloaded.herokuapp.com/badge?page_id=neongeckocom.neon-tts-plugin-coqui"
+coquiTTS = CoquiTTS()
+
+
+# ChatGPT
+def chat_hf(audio, custom_token, language):
+    try:
+        whisper_text = translate(audio)
+        if whisper_text == "ERROR: You have to either use the microphone or upload an audio file":
+            gpt_response = "MISSING AUDIO: Record your voice by clicking the microphone button, do not forget to stop recording before sending your message ;)"
+        else:
+            #gpt_response = chatgpt(whisper_text, [], fn_index=0)
+            #print(gpt_response)
+            #gpt_response = gpt_response[0]
+            gpt_response = openai_create(whisper_text)
+
+    except:
+        whisper_text = translate(audio)
+        gpt_response = """Sorry, I'm quite busy right now, but please try again later :)"""
+    
+    # to voice
+    with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as fp:
+        coquiTTS.get_tts(gpt_response, fp, speaker = {"language" : language})
+
+    return whisper_text, gpt_response, fp.name
+
+# whisper
+#def translate(audio):
+#    print("""
+#    —
+#    Sending audio to Whisper ...
+#    —
+#    """)
+#    
+#    audio = whisper.load_audio(audio)
+#    audio = whisper.pad_or_trim(audio)
+#    
+#    mel = whisper.log_mel_spectrogram(audio).to(whisper_model.device)
+#    
+#    _, probs = whisper_model.detect_language(mel)
+#    
+#    transcript_options = whisper.DecodingOptions(task="transcribe", fp16 = False)
+#    
+#    transcription = whisper.decode(whisper_model, mel, transcript_options)
+#    
+#    print("language spoken: " + transcription.language)
+#    print("transcript: " + transcription.text)
+#    print("———————————————————————————————————————————")  
+#      
+#    return transcription.text
+
+def translate(audio):
+    print("""
+    —
+    Sending audio to Whisper ...
+    —
+    """)
+    #_, text_result = whisper(audio, "", fn_index=0)
+    text_result = whisper(audio, None, "transcribe", fn_index=0)
+    print(text_result)
+    return text_result
+
+
+def openai_create(prompt):
+
+    response = openai.Completion.create(
+    model="text-davinci-003",
+    prompt=prompt,
+    temperature=0.9,
+    max_tokens=150,
+    top_p=1,
+    frequency_penalty=0,
+    presence_penalty=0.6,
+    stop=[" Human:", " AI:"]
+    )
+    print(response.choices[0].text)
+    return response.choices[0].text
+
+with gr.Blocks() as blocks:
+    gr.Markdown("<h1 style='text-align: center; margin-bottom: 1rem'>"
+                + title
+                + "</h1>")
+    #gr.Markdown(description)
+    radio = gr.Radio(label="Language", choices=LANGUAGES, value=default_lang)
+    with gr.Row(equal_height=True):# equal_height=False
+        with gr.Column():# variant="panel"
+            audio_file = gr.Audio(source="microphone", type="filepath")
+            custom_token = gr.Textbox(label='If it fails, use your own session token', placeholder="your own session token")
+            with gr.Row():# mobile_collapse=False
+                submit = gr.Button("Submit", variant="primary")
+        with gr.Column():
+            text1 = gr.Textbox(label="Speech to Text")
+            text2 = gr.Textbox(label="ChatGPT Response")
+            audio = gr.Audio(label="Output", interactive=False)
+    #gr.Markdown(info)
+    #gr.Markdown("<center>"
+    #            +f'<img src={badge} alt="visitors badge"/>'
+    #            +"</center>")
+
+    # actions
+    submit.click(
+        chat_hf,
+        [audio_file, custom_token, radio],
+        [text1, text2, audio],
+    )
+    #radio.change(lambda lang: CoquiTTS.langs[lang]["sentence"], radio, text2)
+
+
+blocks.launch(debug=True)

packages.txt

@@ -0,0 +1,2 @@
+libsndfile1
+espeak-ng

pygpt.py

@@ -0,0 +1,112 @@
+import uuid
+import asyncio
+import socketio
+import datetime
+import json
+import base64
+
+class PyGPT:
+    def __init__(self, session_token, bypass_node='https://gpt.pawan.krd'):
+        self.ready = False
+        self.socket = socketio.AsyncClient()
+        self.socket.on('connect', self.on_connect)
+        self.socket.on('disconnect', self.on_disconnect)
+        self.session_token = session_token
+        self.conversations = []
+        self.auth = None
+        self.expires = datetime.datetime.now()
+        self.pause_token_checks = False
+        self.bypass_node = bypass_node
+        asyncio.create_task(self.cleanup_conversations())
+
+    async def connect(self):
+        await self.socket.connect(self.bypass_node)
+
+    async def disconnect(self):
+        await self.socket.disconnect()
+        await self.socket.close()
+
+    def on_connect(self):
+        print('Connected to server')
+        asyncio.create_task(self.check_tokens())
+
+    def on_disconnect(self):
+        print('Disconnected from server')
+        self.ready = False
+
+    async def check_tokens(self):
+        while True:
+            if self.pause_token_checks:
+                await asyncio.sleep(0.5)
+                continue
+            self.pause_token_checks = True
+            now = datetime.datetime.now()
+            offset = datetime.timedelta(minutes=2)
+            if self.expires < (now - offset) or not self.auth:
+                await self.get_tokens()
+            self.pause_token_checks = False
+            await asyncio.sleep(0.5)
+
+    async def cleanup_conversations(self):
+        while True:
+            await asyncio.sleep(60)
+            now = datetime.datetime.now()
+            self.conversations = [c for c in self.conversations if now - c['last_active'] < datetime.timedelta(minutes=2)]
+
+    def add_conversation(self, id):
+        conversation = {
+            'id': id,
+            'conversation_id': None,
+            'parent_id': uuid.uuid4(),
+            'last_active': datetime.datetime.now()
+        }
+        self.conversations.append(conversation)
+        return conversation
+
+    def get_conversation_by_id(self, id):
+        conversation = next((c for c in self.conversations if c['id'] == id), None)
+        if conversation is None:
+            conversation = self.add_conversation(id)
+        else:
+            conversation['last_active'] = datetime.datetime.now()
+        return conversation
+
+    async def wait_for_ready(self):
+        while not self.ready:
+            await asyncio.sleep(0.025)
+        print('Ready!!')
+
+    async def ask(self, prompt, id='default'):
+        if not self.auth or not self.validate_token(self.auth):
+            await self.get_tokens()
+        conversation = self.get_conversation_by_id(id)
+        data = await self.socket.call('askQuestion', {
+            'prompt': prompt,
+            'parentId': str(conversation['parent_id']),
+            'conversationId': str(conversation['conversation_id']),
+            'auth': self.auth
+        })
+
+        if 'error' in data:
+            print(f'Error: {data["error"]}')
+        conversation['parent_id'] = data['messageId']
+        conversation['conversation_id'] = data['conversationId']
+        return data['answer']
+
+    def validate_token(self, token):
+        if not token:
+            return False
+        parsed = json.loads(base64.b64decode(f'{token.split(".")[1]}==').decode())
+        return datetime.datetime.now() <= datetime.datetime.fromtimestamp(parsed['exp'])
+
+    async def get_tokens(self):
+        await asyncio.sleep(1)
+        data = await self.socket.call('getSession', self.session_token)
+
+        if 'error' in data:
+            print(f'Error getting session: {data["error"]}')
+        else:
+            self.auth = data['auth']
+            self.expires = datetime.datetime.strptime(data['expires'], '%Y-%m-%dT%H:%M:%S.%fZ')
+            self.session_token = data['sessionToken']
+            self.ready = True

requirements.txt

@@ -0,0 +1,3 @@
+neon-tts-plugin-coqui==0.7.0
+openai
+PyGPT