added gradio and openai api threading

app.ipynb

@@ -67,7 +67,16 @@
     "import gradio as gr\n",
     "import openai\n",
     "from pydub import AudioSegment\n",
-    "import io"
+    "import io\n",
+    "from datetime import datetime\n",
+    "from math import ceil\n",
+    "from multiprocessing.pool import ThreadPool\n",
+    "from functools import partial\n",
+    "from tenacity import (\n",
+    "    retry,\n",
+    "    stop_after_attempt,\n",
+    "    wait_random_exponential,\n",
+    ")  # for exponential backoff"
    ]
   },
   {
@@ -96,6 +105,29 @@
     "tts_voices = ['alloy', 'echo', 'fable', 'onyx', 'nova', 'shimmer']"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8eb7e7d5-7121-4762-b8d1-e5a9539e2b36",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "clean_text_prompt = \"\"\"Your job is to clean up text that is going to be fed into a text to speech (TTS) model. You must remove parts of the text that would not normally be spoken such as reference marks `[1]`, spurious citations such as `(Reddy et al., 2021; Wu et al., 2022; Chang et al., 2022; Kondratyuk et al., 2023)` and any other part of the text that is not normally spoken. Please also clean up sections and headers so they are on new lines with proper numbering. You must also clean up any math formulas that are salvageable from being copied from a scientific paper. If they are garbled and do not make sense then remove them. You must carefully perform the text cleanup so it is translated into speech that is easy to listen to however you must not modify the text otherwise. It is critical that you repeat all of the text without modifications except for the cleanup activities you've been instructed to do. Also you must clean all of the text you are given, you may not omit any of it or stop the cleanup task early.\"\"\"\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "52d373be-3a79-412e-8ca2-92bb443fa52d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "#Number of threads created PER USER REQUEST. This throttels the # of API requests PER USER request. This is in ADDITION to the Gradio threads.\n",
+    "OPENAI_CLIENT_TTS_THREADS = 10 "
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -175,6 +207,67 @@
     "        return combined_mp3.getvalue()"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4691703d-ed0f-4481-8006-b2906289b780",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "def create_speech_openai(chunk_idx, input, model='tts-1', voice='alloy', speed=1.0, **kwargs):\n",
+    "    client = openai.OpenAI()\n",
+    "    \n",
+    "    @retry(wait=wait_random_exponential(min=1, max=180), stop=stop_after_attempt(6))\n",
+    "    def _create_speech_with_backoff(**kwargs):\n",
+    "        return client.audio.speech.create(**kwargs)\n",
+    "    \n",
+    "    response = _create_speech_with_backoff(input=input, model=model, voice=voice, speed=speed, **kwargs)\n",
+    "    client.close()\n",
+    "    return chunk_idx, response.content"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e34bb4aa-698c-4452-8cda-bd02b38f7122",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "def create_speech2(input_text, model='tts-1', voice='alloy', progress=gr.Progress(), **kwargs):\n",
+    "    start = datetime.now()\n",
+    "    # Split the input text into chunks\n",
+    "    chunks = split_text(input_text)\n",
+    "\n",
+    "    # Initialize the progress bar\n",
+    "    progress(0, desc=f\"Started processing {len(chunks)} text chunks using {OPENAI_CLIENT_TTS_THREADS} threads. ETA is ~{ceil(len(chunks)/OPENAI_CLIENT_TTS_THREADS)} min.\")\n",
+    "\n",
+    "    # Initialize a list to hold the audio data of each chunk\n",
+    "    audio_data = []\n",
+    "\n",
+    "    # Process each chunk\n",
+    "    with ThreadPool(processes=OPENAI_CLIENT_TTS_THREADS) as pool:\n",
+    "        results = pool.starmap(\n",
+    "            partial(create_speech_openai, model=model, voice=voice, **kwargs), \n",
+    "            zip(range(len(chunks)),chunks)\n",
+    "        )\n",
+    "    audio_data = [o[1] for o in sorted(results)]\n",
+    "\n",
+    "    # Progress\n",
+    "    progress(.9, desc=f\"Merging audio chunks... {(datetime.now()-start).seconds} seconds to process.\")\n",
+    "    \n",
+    "    # Concatenate the audio data from all chunks\n",
+    "    combined_audio = concatenate_mp3(audio_data)\n",
+    "\n",
+    "    # Final update to the progress bar\n",
+    "    progress(1, desc=f\"Processing completed... {(datetime.now()-start).seconds} seconds to process.\")\n",
+    "    \n",
+    "    print(f\"Processing time: {(datetime.now()-start).seconds} seconds.\")\n",
+    "\n",
+    "    return combined_audio\n"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -264,7 +357,8 @@
     "#| export\n",
     "with gr.Blocks(title='OpenAI TTS', head='OpenAI TTS') as app:\n",
     "    gr.Markdown(\"# OpenAI TTS\")\n",
-    "    gr.Markdown(\"Start typing below and then click **Go** to create the speech from your text. The current limit is 4,000 characters.\")\n",
+    "    gr.Markdown(\"\"\"Start typing below and then click **Go** to create the speech from your text. The current limit is 4,000 characters. \n",
+    "For requests longer than 4,000 chars they will be broken into chunks of 4,000 or less chars automatically.\"\"\")\n",
     "    with gr.Row():\n",
     "        input_text = gr.Textbox(max_lines=100, label=\"Enter text here\")\n",
     "    with gr.Row():\n",
@@ -277,7 +371,7 @@
     "    input_text.input(fn=get_generation_cost, inputs=[input_text,tts_model_dropdown], outputs=generation_cost)\n",
     "    tts_model_dropdown.input(fn=get_generation_cost, inputs=[input_text,tts_model_dropdown], outputs=generation_cost)\n",
     "    go_btn = gr.Button(\"Go\")\n",
-    "    go_btn.click(fn=create_speech, inputs=[input_text, tts_model_dropdown, tts_voice_dropdown], outputs=[output_audio])\n",
+    "    go_btn.click(fn=create_speech2, inputs=[input_text, tts_model_dropdown, tts_voice_dropdown], outputs=[output_audio])\n",
     "    clear_btn = gr.Button('Clear')\n",
     "    clear_btn.click(fn=lambda: '', outputs=input_text)\n",
     "    "
@@ -292,7 +386,8 @@
    "source": [
     "#| export\n",
     "launch_kwargs = {'auth':('username',GRADIO_PASSWORD),\n",
-    "                 'auth_message':'Please log in to Mat\\'s TTS App with username: username and password.'}"
+    "                 'auth_message':'Please log in to Mat\\'s TTS App with username: username and password.'}\n",
+    "queue_kwargs = {'default_concurrency_limit':10}"
    ]
   },
   {
@@ -304,6 +399,7 @@
    "source": [
     "#| hide\n",
     "#Notebook launch\n",
+    "app.queue(**queue_kwargs)\n",
     "app.launch(**launch_kwargs)"
    ]
   },
@@ -317,6 +413,7 @@
     "#| export\n",
     "#.py launch\n",
     "if __name__ == \"__main__\":\n",
+    "    app.queue(**queue_kwargs)\n",
     "    app.launch(**launch_kwargs)"
    ]
   },

app.py

@@ -1,7 +1,8 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: app.ipynb.
 
 # %% auto 0
-__all__ = ['secret_import_failed', 'tts_voices', 'launch_kwargs', 'split_text', 'concatenate_mp3', 'create_speech',
+__all__ = ['secret_import_failed', 'tts_voices', 'clean_text_prompt', 'OPENAI_CLIENT_TTS_THREADS', 'launch_kwargs',
+           'queue_kwargs', 'split_text', 'concatenate_mp3', 'create_speech_openai', 'create_speech2', 'create_speech',
            'get_input_text_len', 'get_generation_cost']
 
 # %% app.ipynb 1
@@ -33,6 +34,15 @@ import gradio as gr
 import openai
 from pydub import AudioSegment
 import io
+from datetime import datetime
+from math import ceil
+from multiprocessing.pool import ThreadPool
+from functools import partial
+from tenacity import (
+    retry,
+    stop_after_attempt,
+    wait_random_exponential,
+)  # for exponential backoff
 
 # %% app.ipynb 4
 try:
@@ -45,6 +55,14 @@ except:
 tts_voices = ['alloy', 'echo', 'fable', 'onyx', 'nova', 'shimmer']
 
 # %% app.ipynb 6
+clean_text_prompt = """Your job is to clean up text that is going to be fed into a text to speech (TTS) model. You must remove parts of the text that would not normally be spoken such as reference marks `[1]`, spurious citations such as `(Reddy et al., 2021; Wu et al., 2022; Chang et al., 2022; Kondratyuk et al., 2023)` and any other part of the text that is not normally spoken. Please also clean up sections and headers so they are on new lines with proper numbering. You must also clean up any math formulas that are salvageable from being copied from a scientific paper. If they are garbled and do not make sense then remove them. You must carefully perform the text cleanup so it is translated into speech that is easy to listen to however you must not modify the text otherwise. It is critical that you repeat all of the text without modifications except for the cleanup activities you've been instructed to do. Also you must clean all of the text you are given, you may not omit any of it or stop the cleanup task early."""
+
+
+# %% app.ipynb 7
+#Number of threads created PER USER REQUEST. This throttels the # of API requests PER USER request. This is in ADDITION to the Gradio threads.
+OPENAI_CLIENT_TTS_THREADS = 10 
+
+# %% app.ipynb 8
 def split_text(input_text, max_length=4000, lookback=1000):
     # If the text is shorter than the max_length, return it as is
     if len(input_text) <= max_length:
@@ -77,7 +95,7 @@ def split_text(input_text, max_length=4000, lookback=1000):
 
     return chunks
 
-# %% app.ipynb 7
+# %% app.ipynb 9
 def concatenate_mp3(mp3_files):
     if len(mp3_files) == 1:
         return mp3_files[0]
@@ -107,7 +125,53 @@ def concatenate_mp3(mp3_files):
 
         return combined_mp3.getvalue()
 
-# %% app.ipynb 8
+# %% app.ipynb 10
+def create_speech_openai(chunk_idx, input, model='tts-1', voice='alloy', speed=1.0, **kwargs):
+    client = openai.OpenAI()
+    
+    @retry(wait=wait_random_exponential(min=1, max=180), stop=stop_after_attempt(6))
+    def _create_speech_with_backoff(**kwargs):
+        return client.audio.speech.create(**kwargs)
+    
+    response = _create_speech_with_backoff(input=input, model=model, voice=voice, speed=speed, **kwargs)
+    client.close()
+    return chunk_idx, response.content
+
+# %% app.ipynb 11
+def create_speech2(input_text, model='tts-1', voice='alloy', progress=gr.Progress(), **kwargs):
+    start = datetime.now()
+    # Split the input text into chunks
+    chunks = split_text(input_text)
+
+    # Initialize the progress bar
+    progress(0, desc=f"Started processing {len(chunks)} text chunks using {OPENAI_CLIENT_TTS_THREADS} threads. ETA is ~{ceil(len(chunks)/OPENAI_CLIENT_TTS_THREADS)} min.")
+
+    # Initialize a list to hold the audio data of each chunk
+    audio_data = []
+
+    # Process each chunk
+    with ThreadPool(processes=OPENAI_CLIENT_TTS_THREADS) as pool:
+        results = pool.starmap(
+            partial(create_speech_openai, model=model, voice=voice, **kwargs), 
+            zip(range(len(chunks)),chunks)
+        )
+    audio_data = [o[1] for o in sorted(results)]
+
+    # Progress
+    progress(.9, desc=f"Merging audio chunks... {(datetime.now()-start).seconds} seconds to process.")
+    
+    # Concatenate the audio data from all chunks
+    combined_audio = concatenate_mp3(audio_data)
+
+    # Final update to the progress bar
+    progress(1, desc=f"Processing completed... {(datetime.now()-start).seconds} seconds to process.")
+    
+    print(f"Processing time: {(datetime.now()-start).seconds} seconds.")
+
+    return combined_audio
+
+
+# %% app.ipynb 12
 def create_speech(input_text, model='tts-1', voice='alloy', progress=gr.Progress()):
     # Split the input text into chunks
     chunks = split_text(input_text)
@@ -150,11 +214,11 @@ def create_speech(input_text, model='tts-1', voice='alloy', progress=gr.Progress
     return combined_audio
 
 
-# %% app.ipynb 9
+# %% app.ipynb 13
 def get_input_text_len(input_text):
     return len(input_text)
 
-# %% app.ipynb 10
+# %% app.ipynb 14
 def get_generation_cost(input_text, tts_model_dropdown):
     text_len = len(input_text)
     if tts_model_dropdown.endswith('-hd'):
@@ -163,10 +227,11 @@ def get_generation_cost(input_text, tts_model_dropdown):
         cost = text_len/1000 * 0.015
     return "${:,.3f}".format(cost)
 
-# %% app.ipynb 11
+# %% app.ipynb 15
 with gr.Blocks(title='OpenAI TTS', head='OpenAI TTS') as app:
     gr.Markdown("# OpenAI TTS")
-    gr.Markdown("Start typing below and then click **Go** to create the speech from your text. The current limit is 4,000 characters.")
+    gr.Markdown("""Start typing below and then click **Go** to create the speech from your text. The current limit is 4,000 characters. 
+For requests longer than 4,000 chars they will be broken into chunks of 4,000 or less chars automatically.""")
     with gr.Row():
         input_text = gr.Textbox(max_lines=100, label="Enter text here")
     with gr.Row():
@@ -179,16 +244,18 @@ with gr.Blocks(title='OpenAI TTS', head='OpenAI TTS') as app:
     input_text.input(fn=get_generation_cost, inputs=[input_text,tts_model_dropdown], outputs=generation_cost)
     tts_model_dropdown.input(fn=get_generation_cost, inputs=[input_text,tts_model_dropdown], outputs=generation_cost)
     go_btn = gr.Button("Go")
-    go_btn.click(fn=create_speech, inputs=[input_text, tts_model_dropdown, tts_voice_dropdown], outputs=[output_audio])
+    go_btn.click(fn=create_speech2, inputs=[input_text, tts_model_dropdown, tts_voice_dropdown], outputs=[output_audio])
     clear_btn = gr.Button('Clear')
     clear_btn.click(fn=lambda: '', outputs=input_text)
     
 
-# %% app.ipynb 12
+# %% app.ipynb 16
 launch_kwargs = {'auth':('username',GRADIO_PASSWORD),
                  'auth_message':'Please log in to Mat\'s TTS App with username: username and password.'}
+queue_kwargs = {'default_concurrency_limit':10}
 
-# %% app.ipynb 14
+# %% app.ipynb 18
 #.py launch
 if __name__ == "__main__":
+    app.queue(**queue_kwargs)
     app.launch(**launch_kwargs)

requirements.txt

@@ -1,3 +1,4 @@
 openai==1.10.0
 gradio==4.16.0
 pydub==0.25.1
+tenacity==8.2.3