Major pull from github

Added Docker header in README.md

Removed png assets

APIs.py

@@ -9,25 +9,15 @@ from retrying import retry
  
 
 os.environ['OPENBLAS_NUM_THREADS'] = '1'
-
 SAMPLE_RATE = 32000
 
 
 with open('config.yaml', 'r') as file:
     config = yaml.safe_load(file)
-    tts_port = config['Text-to-Speech']['service-port']
-    ttm_port = config['Text-to-Music']['service-port']
-    tta_port = config['Text-to-Audio']['service-port']
-    sr_port = config['Speech-Restoration']['service-port']
-    vp_port = config['Voice-Parser']['service-port']
+    service_port = config['Service-Port']
     enable_sr = config['Speech-Restoration']['Enable']
 
 
-def IDLE(length=1.0, out_wav='out.wav', sr=SAMPLE_RATE):
-    idle = np.zeros(int(length * sr))
-    WRITE_AUDIO(idle, name=out_wav, sr=SAMPLE_RATE)
-
-
 def LOUDNESS_NORM(audio, sr=32000, volumn=-25):
     # peak normalize audio to -1 dB
     peak_normalized_audio = pyln.normalize.peak(audio, -10.0)
@@ -57,7 +47,7 @@ def WRITE_AUDIO(wav, name=None, sr=SAMPLE_RATE):
     if max_value > 1:
         wav *= 0.9 / max_value
     
-    # print(f'WRITE_AUDIO to {name}')
+    # write audio
     write(name, sr, np.round(wav*32767).astype(np.int16))
 
 
@@ -81,10 +71,6 @@ def MIX(wavs=[['1.wav', 0.], ['2.wav', 10.]], out_wav='out.wav', sr=SAMPLE_RATE)
     wavs:[[wav_name, absolute_offset], ...]
     """
 
-    # last_name, last_offset = wavs[-1]
-    # last_len = len(READ_AUDIO_NUMPY(last_name))
-    # max_length = int(last_offset * sr + last_len)
-
     max_length = max([int(wav[1]*sr + len(READ_AUDIO_NUMPY(wav[0]))) for wav in wavs])
     template_wav = np.zeros(max_length)
 
@@ -125,7 +111,7 @@ def COMPUTE_LEN(wav):
 
 @retry(stop_max_attempt_number=5, wait_fixed=2000)
 def TTM(text, length=10, volume=-28, out_wav='out.wav'):
-    url = f'http://127.0.0.1:{ttm_port}/generate_music'
+    url = f'http://127.0.0.1:{service_port}/generate_music'
     data = {
         'text': f'{text}',
         'length': f'{length}',
@@ -143,7 +129,7 @@ def TTM(text, length=10, volume=-28, out_wav='out.wav'):
 
 @retry(stop_max_attempt_number=5, wait_fixed=2000)
 def TTA(text, length=5, volume=-35, out_wav='out.wav'):
-    url = f'http://127.0.0.1:{tta_port}/generate_audio'
+    url = f'http://127.0.0.1:{service_port}/generate_audio'
     data = {
         'text': f'{text}',
         'length': f'{length}',
@@ -162,7 +148,7 @@ def TTA(text, length=5, volume=-35, out_wav='out.wav'):
 
 @retry(stop_max_attempt_number=5, wait_fixed=2000)
 def TTS(text, speaker='news_anchor', volume=-20, out_wav='out.wav', enhanced=enable_sr, speaker_id='', speaker_npz=''):
-    url = f'http://127.0.0.1:{tts_port}/generate_speech'
+    url = f'http://127.0.0.1:{service_port}/generate_speech'
     data = {
     'text': f'{text}',
     'speaker_id': f'{speaker_id}',
@@ -185,7 +171,7 @@ def TTS(text, speaker='news_anchor', volume=-20, out_wav='out.wav', enhanced=ena
 
 @retry(stop_max_attempt_number=5, wait_fixed=2000)
 def SR(processfile):
-    url = f'http://127.0.0.1:{sr_port}/fix_audio'
+    url = f'http://127.0.0.1:{service_port}/fix_audio'
     data = {'processfile': f'{processfile}'}
 
     response = requests.post(url, json=data)
@@ -199,7 +185,7 @@ def SR(processfile):
 
 @retry(stop_max_attempt_number=5, wait_fixed=2000)
 def VP(wav_path, out_dir):
-    url = f'http://127.0.0.1:{vp_port}/parse_voice'
+    url = f'http://127.0.0.1:{service_port}/parse_voice'
     data = {
         'wav_path': f'{wav_path}', 
         'out_dir':f'{out_dir}'

AudioCraft/app.py

@@ -1,110 +0,0 @@
-import sys
-sys.path.append('../AudioJourney')
-import os
-import yaml
-import logging
-import torchaudio
-from APIs import WRITE_AUDIO, LOUDNESS_NORM
-from utils import fade
-from flask import Flask, request, jsonify
-
-with open('config.yaml', 'r') as file:
-    config = yaml.safe_load(file)
-
-# Configure the logging format and level
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(levelname)s - %(message)s'
-)
-
-# Create a FileHandler for the log file
-os.makedirs('services_logs', exist_ok=True)
-log_filename = 'services_logs/Text-to-Audio-Music.log'
-file_handler = logging.FileHandler(log_filename, mode='w')
-file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
-
-# Add the FileHandler to the root logger
-logging.getLogger('').addHandler(file_handler)
-
-
-# Initialize the model here
-from audiocraft.models import AudioGen, MusicGen
-tta_model = AudioGen.get_pretrained('facebook/audiogen-medium')
-logging.info('AudioGen is loaded ...')
-
-model_size = config['Text-to-Music']['model_size']
-ttm_model = MusicGen.get_pretrained(f'facebook/musicgen-{model_size}')
-logging.info(f'MusicGen ({model_size}) is loaded ...')
-
-app = Flask(__name__)
-
-@app.route('/generate_audio', methods=['POST'])
-def generate_audio():
-    # Receive the text from the POST request
-    data = request.json
-    text = data['text']
-    length = float(data.get('length', 5.0))
-    volume = float(data.get('volume', -35))
-    output_wav = data.get('output_wav', 'out.wav')
-
-    logging.info(f'TTA (AudioGen): Prompt: {text}, length: {length} seconds, volume: {volume} dB')
-    
-    try:
-        tta_model.set_generation_params(duration=length)  
-        wav = tta_model.generate([text])  
-        wav = torchaudio.functional.resample(wav, orig_freq=16000, new_freq=32000)
-
-        wav = wav.squeeze().cpu().detach().numpy()
-        wav = fade(LOUDNESS_NORM(wav, volumn=volume))
-        WRITE_AUDIO(wav, name=output_wav)
-
-        # Return success message and the filename of the generated audio
-        return jsonify({'message': f'Text-to-Audio generated successfully | {text}', 'file': output_wav})
-
-    except Exception as e:
-        return jsonify({'API error': str(e)}), 500
-
-
-@app.route('/generate_music', methods=['POST'])
-def generate_music():
-    # Receive the text from the POST request
-    data = request.json
-    text = data['text']
-    length = float(data.get('length', 5.0))
-    volume = float(data.get('volume', -35))
-    output_wav = data.get('output_wav', 'out.wav')
-
-    logging.info(f'TTM (MusicGen): Prompt: {text}, length: {length} seconds, volume: {volume} dB')
-
-
-    try:
-        ttm_model.set_generation_params(duration=length)  
-        wav = ttm_model.generate([text])  
-        wav = wav[0][0].cpu().detach().numpy()
-        wav = fade(LOUDNESS_NORM(wav, volumn=volume))
-        WRITE_AUDIO(wav, name=output_wav)
-
-        # Return success message and the filename of the generated audio
-        return jsonify({'message': f'Text-to-Music generated successfully | {text}', 'file': output_wav})
-
-    except Exception as e:
-        # Return error message if something goes wrong
-        return jsonify({'API error': str(e)}), 500
-
-
-if __name__ == '__main__':
-    import yaml
-    with open('config.yaml', 'r') as file:
-        config = yaml.safe_load(file)
-
-    tta_service_port = config['Text-to-Audio']['service-port']
-    ttm_service_port = config['Text-to-Audio']['service-port']
-
-    if tta_service_port != ttm_service_port:
-        msg = 'Ports of TTA and TTM should be same if you are using Audiocraft ...'
-        logging.info(msg)
-        raise ValueError(msg)
-
-    app.run(debug=False, port=tta_service_port)
-
-

Dockerfile

@@ -0,0 +1,28 @@
+FROM python:3.11
+
+# Install miniconda
+RUN apt-get install -y wget && rm -rf /var/lib/apt/lists/*
+
+RUN wget \
+	https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+	&& bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/miniconda3 \
+	&& rm -f Miniconda3-latest-Linux-x86_64.sh
+
+# Add conda binary to PATH variable
+ENV HOME=/home/user \
+	PATH=/opt/miniconda3/bin:/home/user/.local/bin:$PATH \
+	CONDA_PREFIX=/opt/miniconda3/envs
+
+# Setup conda envs
+WORKDIR $HOME/app
+COPY . .
+
+# Conda envs setup 
+RUN bash ./scripts/EnvsSetup.sh
+
+# pre-download all models
+RUN conda run --live-stream -n WavJourney python scripts/download_models.py
+RUN mkdir $HOME/app/services_logs
+
+# entrypoint
+ENTRYPOINT bash /home/user/app/scripts/start_service_and_ui.sh

Envs/AudioCraft.yml

@@ -1,4 +1,4 @@
-name: AudioCraft
+name: WavJourney
 channels:
   - nvidia/label/cuda-11.8.0
   - conda-forge

Envs/Bark.yml

@@ -1,4 +1,4 @@
-name: Bark
+name: WavJourney
 channels:
   - conda-forge
   - defaults
@@ -177,4 +177,3 @@ dependencies:
       - xxhash==3.3.0
       - yarl==1.9.2
       - zipp==3.16.1
-prefix: /home/zzk/Workspace/miniconda3/envs/Bark

EnvsSetup/AudioCraft.sh

@@ -1,16 +0,0 @@
-conda env create -f Envs/AudioCraft.yml
-conda run --live-stream -n AudioCraft pip install -U git+https://git@github.com/facebookresearch/audiocraft@c5157b5bf14bf83449c17ea1eeb66c19fb4bc7f0#egg=audiocraft
-# Could not load library libcudnn_cnn_infer.so.8.
-# Error: libnvrtc.so: cannot open shared object file: No such file or directory
-CONDAENV=AudioCraft
-source activate ${CONDAENV}
-conda install -c "nvidia/label/cuda-11.8.0" cuda-toolkit
-python3 -m pip install nvidia-cudnn-cu11==8.5.0.96
-source deactivate
-mkdir -p $CONDA_PREFIX/envs/${CONDAENV}/etc/conda/activate.d
-echo 'CUDNN_PATH=$(dirname $(python -c "import nvidia.cudnn;print(nvidia.cudnn.__file__)"))' >> $CONDA_PREFIX/envs/${CONDAENV}/etc/conda/activate.d/env_vars.sh
-echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CONDA_PREFIX/lib/:$CUDNN_PATH/lib' >> $CONDA_PREFIX/envs/${CONDAENV}/etc/conda/activate.d/env_vars.sh
-source $CONDA_PREFIX/envs/${CONDAENV}/etc/conda/activate.d/env_vars.sh
-
-# If you're using WSL2, you can add the following into ~/.bashrc
-# export LD_LIBRARY_PATH=/usr/lib/wsl/lib:$LD_LIBRARY_PATH

EnvsSetup/Bark.sh

@@ -1 +0,0 @@
-conda env create -f Envs/Bark.yml

EnvsSetup/VoiceFixer.sh

@@ -1 +0,0 @@
-conda env create -f Envs/VoiceFixer.yml

EnvsSetup/WavJourney.sh

@@ -1 +0,0 @@
-conda env create -f Envs/WavJourney.yml

LICENSE

@@ -0,0 +1,251 @@
+Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International Public License
+
+By exercising the Licensed Rights (defined below), You accept and agree to be
+bound by the terms and conditions of this Creative Commons
+Attribution-NonCommercial-NoDerivatives 4.0 International Public License
+("Public License"). To the extent this Public License may be interpreted as a
+contract, You are granted the Licensed Rights in consideration of Your
+acceptance of these terms and conditions, and the Licensor grants You such
+rights in consideration of benefits the Licensor receives from making the
+Licensed Material available under these terms and conditions.
+
+Section 1 – Definitions.
+
+    a. Adapted Material means material subject to Copyright and Similar Rights
+       that is derived from or based upon the Licensed Material and in which
+       the Licensed Material is translated, altered, arranged, transformed, or
+       otherwise modified in a manner requiring permission under the Copyright
+       and Similar Rights held by the Licensor. For purposes of this Public
+       License, where the Licensed Material is a musical work, performance, or
+       sound recording, Adapted Material is always produced where the Licensed
+       Material is synched in timed relation with a moving image.
+    b. Copyright and Similar Rights means copyright and/or similar rights
+       closely related to copyright including, without limitation,
+       performance, broadcast, sound recording, and Sui Generis Database
+       Rights, without regard to how the rights are labeled or categorized.
+       For purposes of this Public License, the rights specified in Section
+       2(b)(1)-(2) are not Copyright and Similar Rights.
+    c. Effective Technological Measures means those measures that, in the
+       absence of proper authority, may not be circumvented under laws
+       fulfilling obligations under Article 11 of the WIPO Copyright Treaty
+       adopted on December 20, 1996, and/or similar international agreements.
+    d. Exceptions and Limitations means fair use, fair dealing, and/or any
+       other exception or limitation to Copyright and Similar Rights that
+       applies to Your use of the Licensed Material.
+    e. Licensed Material means the artistic or literary work, database, or
+       other material to which the Licensor applied this Public License.
+    f. Licensed Rights means the rights granted to You subject to the terms
+       and conditions of this Public License, which are limited to all
+       Copyright and Similar Rights that apply to Your use of the Licensed
+       Material and that the Licensor has authority to license.
+    g. Licensor means the individual(s) or entity(ies) granting rights under
+       this Public License.
+    h. NonCommercial means not primarily intended for or directed towards
+       commercial advantage or monetary compensation. For purposes of this
+       Public License, the exchange of the Licensed Material for other
+       material subject to Copyright and Similar Rights by digital
+       file-sharing or similar means is NonCommercial provided there is no
+       payment of monetary compensation in connection with the exchange.
+    i. Share means to provide material to the public by any means or process
+       that requires permission under the Licensed Rights, such as
+       reproduction, public display, public performance, distribution,
+       dissemination, communication, or importation, and to make material
+       available to the public including in ways that members of the public
+       may access the material from a place and at a time individually chosen
+       by them.
+    j. Sui Generis Database Rights means rights other than copyright resulting
+       from Directive 96/9/EC of the European Parliament and of the Council of
+       11 March 1996 on the legal protection of databases, as amended and/or
+       succeeded, as well as other essentially equivalent rights anywhere in
+       the world.
+    k. You means the individual or entity exercising the Licensed Rights under
+       this Public License. Your has a corresponding meaning.
+
+Section 2 – Scope.
+
+    a. License grant.
+        1. Subject to the terms and conditions of this Public License, the
+           Licensor hereby grants You a worldwide, royalty-free,
+           non-sublicensable, non-exclusive, irrevocable license to exercise
+           the Licensed Rights in the Licensed Material to:
+            A. reproduce and Share the Licensed Material, in whole or in part,
+               for NonCommercial purposes only; and
+            B. produce and reproduce, but not Share, Adapted Material for
+               NonCommercial purposes only.
+        2. Exceptions and Limitations. For the avoidance of doubt, where
+           Exceptions and Limitations apply to Your use, this Public License
+           does not apply, and You do not need to comply with its terms and
+           conditions.
+        3. Term. The term of this Public License is specified in Section 6(a).
+        4. Media and formats; technical modifications allowed. The Licensor
+           authorizes You to exercise the Licensed Rights in all media and
+           formats whether now known or hereafter created, and to make
+           technical modifications necessary to do so. The Licensor waives
+           and/or agrees not to assert any right or authority to forbid You
+           from making technical modifications necessary to exercise the
+           Licensed Rights, including technical modifications necessary to
+           circumvent Effective Technological Measures. For purposes of this
+           Public License, simply making modifications authorized by this
+           Section 2(a)(4) never produces Adapted Material.
+        5. Downstream recipients.
+            A. Offer from the Licensor – Licensed Material. Every recipient of
+               the Licensed Material automatically receives an offer from the
+               Licensor to exercise the Licensed Rights under the terms and
+               conditions of this Public License.
+            B. No downstream restrictions. You may not offer or impose any
+               additional or different terms or conditions on, or apply any
+               Effective Technological Measures to, the Licensed Material if
+               doing so restricts exercise of the Licensed Rights by any
+               recipient of the Licensed Material.
+        6. No endorsement. Nothing in this Public License constitutes or may
+           be construed as permission to assert or imply that You are, or that
+           Your use of the Licensed Material is, connected with, or sponsored,
+           endorsed, or granted official status by, the Licensor or others
+           designated to receive attribution as provided in Section
+           3(a)(1)(A)(i).
+
+    b. Other rights.
+        1. Moral rights, such as the right of integrity, are not licensed
+           under this Public License, nor are publicity, privacy, and/or other
+           similar personality rights; however, to the extent possible, the
+           Licensor waives and/or agrees not to assert any such rights held by
+           the Licensor to the limited extent necessary to allow You to
+           exercise the Licensed Rights, but not otherwise.
+        2. Patent and trademark rights are not licensed under this Public
+           License.
+        3. To the extent possible, the Licensor waives any right to collect
+           royalties from You for the exercise of the Licensed Rights, whether
+           directly or through a collecting society under any voluntary or
+           waivable statutory or compulsory licensing scheme. In all other
+           cases the Licensor expressly reserves any right to collect such
+           royalties, including when the Licensed Material is used other than
+           for NonCommercial purposes.
+
+Section 3 – License Conditions.
+
+Your exercise of the Licensed Rights is expressly made subject to the following conditions.
+
+    a. Attribution.
+
+        1. If You Share the Licensed Material, You must:
+            A. retain the following if it is supplied by the Licensor with the
+               Licensed Material:
+                i. identification of the creator(s) of the Licensed Material
+                   and any others designated to receive attribution, in any
+                   reasonable manner requested by the Licensor (including by
+                   pseudonym if designated);
+                ii. a copyright notice;
+                iii. a notice that refers to this Public License;
+                iv. a notice that refers to the disclaimer of warranties;
+                v. a URI or hyperlink to the Licensed Material to the extent
+                   reasonably practicable;
+            B. indicate if You modified the Licensed Material and retain an
+               indication of any previous modifications; and
+            C. indicate the Licensed Material is licensed under this Public
+               License, and include the text of, or the URI or hyperlink to,
+               this Public License.
+
+            For the avoidance of doubt, You do not have permission under this
+            Public License to Share Adapted Material.
+
+        2. You may satisfy the conditions in Section 3(a)(1) in any reasonable
+           manner based on the medium, means, and context in which You Share
+           the Licensed Material. For example, it may be reasonable to satisfy
+           the conditions by providing a URI or hyperlink to a resource that
+           includes the required information.
+        3. If requested by the Licensor, You must remove any of the
+           information required by Section 3(a)(1)(A) to the extent reasonably
+           practicable.
+
+Section 4 – Sui Generis Database Rights.
+
+Where the Licensed Rights include Sui Generis Database Rights that apply to
+Your use of the Licensed Material:
+
+    a. for the avoidance of doubt, Section 2(a)(1) grants You the right to
+       extract, reuse, reproduce, and Share all or a substantial portion of
+       the contents of the database for NonCommercial purposes only and
+       provided You do not Share Adapted Material;
+    b. if You include all or a substantial portion of the database contents in
+       a database in which You have Sui Generis Database Rights, then the
+       database in which You have Sui Generis Database Rights (but not its
+       individual contents) is Adapted Material; and
+    c. You must comply with the conditions in Section 3(a) if You Share all or
+       a substantial portion of the contents of the database.
+
+For the avoidance of doubt, this Section 4 supplements and does not replace
+Your obligations under this Public License where the Licensed Rights include
+other Copyright and Similar Rights.
+
+Section 5 – Disclaimer of Warranties and Limitation of Liability.
+
+    a. Unless otherwise separately undertaken by the Licensor, to the extent
+       possible, the Licensor offers the Licensed Material as-is and
+       as-available, and makes no representations or warranties of any kind
+       concerning the Licensed Material, whether express, implied, statutory,
+       or other. This includes, without limitation, warranties of title,
+       merchantability, fitness for a particular purpose, non-infringement,
+       absence of latent or other defects, accuracy, or the presence or
+       absence of errors, whether or not known or discoverable. Where
+       disclaimers of warranties are not allowed in full or in part, this
+       disclaimer may not apply to You.
+    b. To the extent possible, in no event will the Licensor be liable to You
+       on any legal theory (including, without limitation, negligence) or
+       otherwise for any direct, special, indirect, incidental, consequential,
+       punitive, exemplary, or other losses, costs, expenses, or damages
+       arising out of this Public License or use of the Licensed Material,
+       even if the Licensor has been advised of the possibility of such
+       losses, costs, expenses, or damages. Where a limitation of liability is
+       not allowed in full or in part, this limitation may not apply to You.
+    c. The disclaimer of warranties and limitation of liability provided above
+       shall be interpreted in a manner that, to the extent possible, most
+       closely approximates an absolute disclaimer and waiver of all
+       liability.
+
+Section 6 – Term and Termination.
+
+    a. This Public License applies for the term of the Copyright and Similar
+       Rights licensed here. However, if You fail to comply with this Public
+       License, then Your rights under this Public License terminate
+       automatically.
+    b. Where Your right to use the Licensed Material has terminated under
+       Section 6(a), it reinstates:
+        1. automatically as of the date the violation is cured, provided it is
+           cured within 30 days of Your discovery of the violation; or
+        2. upon express reinstatement by the Licensor.
+
+      For the avoidance of doubt, this Section 6(b) does not affect any right
+      the Licensor may have to seek remedies for Your violations of this
+      Public License.
+
+    c. For the avoidance of doubt, the Licensor may also offer the Licensed
+       Material under separate terms or conditions or stop distributing the
+       Licensed Material at any time; however, doing so will not terminate
+       this Public License.
+    d. Sections 1, 5, 6, 7, and 8 survive termination of this Public License.
+
+Section 7 – Other Terms and Conditions.
+
+    a. The Licensor shall not be bound by any additional or different terms or
+       conditions communicated by You unless expressly agreed.
+    b. Any arrangements, understandings, or agreements regarding the Licensed
+       Material not stated herein are separate from and independent of the
+       terms and conditions of this Public License.
+
+Section 8 – Interpretation.
+
+    a. For the avoidance of doubt, this Public License does not, and shall not
+       be interpreted to, reduce, limit, restrict, or impose conditions on any
+       use of the Licensed Material that could lawfully be made without
+       permission under this Public License.
+    b. To the extent possible, if any provision of this Public License is
+       deemed unenforceable, it shall be automatically reformed to the minimum
+       extent necessary to make it enforceable. If the provision cannot be
+       reformed, it shall be severed from this Public License without
+       affecting the enforceability of the remaining terms and conditions.
+    c. No term or condition of this Public License will be waived and no
+       failure to comply consented to unless expressly agreed to by the Licensor.
+    d. Nothing in this Public License constitutes or may be interpreted as a
+       limitation upon, or waiver of, any privileges and immunities that apply
+       to the Licensor or You, including from the legal processes of any
+       jurisdiction or authority.

README.md

@@ -1,61 +1,85 @@
----
-title: WavJourney
-emoji: 🔥
-colorFrom: blue
-colorTo: purple
-sdk: gradio
-sdk_version: 3.40.1
-app_file: app.py
-pinned: false
-license: cc-by-nc-nd-4.0
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
-
-# How to run WavJourney?
-1. Install environment by following the bash scripts in `EnvsSetup/`
-2. Start API services; The service logs are in the folder of `logs/`
+# <span style="color: blue;">🎵</span> WavJourney: Compositional Audio Creation with LLMs
+[![arXiv](https://img.shields.io/badge/arXiv-Paper-<COLOR>.svg)](https://arxiv.org/abs/2307.14335) [![GitHub Stars](https://img.shields.io/github/stars/Audio-AGI/WavJourney?style=social)](https://github.com/Audio-AGI/WavJourney/) [![githubio](https://img.shields.io/badge/GitHub.io-Demo_Page-blue?logo=Github&style=flat-square)](https://audio-agi.github.io/WavJourney_demopage/) 
+
+
+This repository contains the official implementation of ["WavJourney: Compositional Audio Creation with Large Language Models"](https://audio-agi.github.io/WavJourney_demopage/WavJourney_arXiv.pdf).
+
+Starting with a text prompt, WavJourney can create audio content with engaging storylines encompassing personalized speakers, lifelike speech in context, emotionally resonant music compositions, and impactful sound effects that enhance the auditory experience. Check the audio examples in the [Project Page](https://audio-agi.github.io/WavJourney_demopage/)!
+
+<!-- <p align="center">
+  <img align="middle" width="800" src="assets/WavJourney.png"/>
+</p> -->
+
+<hr>
+
+
+## Preliminaries
+1. Install the environment:
+```bash
+bash ./scripts/EnvsSetup.sh
+```
+2. Activate the conda environment:
+```bash
+conda activate WavJourney
+```
+
+3. Set your `OpenAI-Key` in `config.yaml` for accessing [GPT-4 API](https://platform.openai.com/account/api-keys) [[Guidance](https://help.openai.com/en/articles/7102672-how-can-i-access-gpt-4)]. Please make sure the 'Service-Port' is not occupied. You can also modify the configuration, check the details described in the configuration file.
+
+3. Pre-download the models (might take some time):
+```bash
+python scripts/download_models.py
+```
+
+5. Start Python API services (e.g., Text-to-Speech, Text-to-Audio)
+```bash
+bash scripts/start_services.sh
+```
+
+## Web APP
  ```bash
- python scripts/start_services.py
+bash scripts/start_ui.sh
   ```
-3. Run AudioJourney client; The results of scripts and audio are in the folder of `output/[datetime]_[instruction text]/`
+
+## Commandline Usage
  ```bash
- conda activate AudioJourney
- python audiojourney_cli.py -f --instruction "News channel BBC broadcast about Trump playing street fighter 6 against Biden"
+ python wavjourney_cli.py -f --input-text "Generate a one-minute introduction to quantum mechanics"
  ```
-4. Kill the API services
- ```bash
-python scripts/kill_services.py
-  ```
 
-5. Start the UI
+
+## Kill the services
+You can kill the running services via this command:
  ```bash
-sh scripts/start_ui.sh
+python scripts/kill_services.py
   ```
-
   
-# Voice Presets
-You can add voice presets to WavJourney to customize the voice actors. Simply provide the voice id, the description and a sample wav file, and WavJourney will pick the voice automatically based on the audio script.
-
-Predefined system voice presets are in `data/voice_presets`, whereas session voice presets are in each session's individual folder. See the example below:
-
-- 📂 **project_folder**
-  - 📂 **data**
-    - 📂 **voice_presets** <-- system voice presets
-      - 📄 **metadata.json** <-- system voice preset metadata
-      - 📂 **npz**
-  - 📂 **output**
-    - 📂 **sessions**
-      - 📂 **session_1**
-        - 📂 **voice_presets** <-- session voice presets
-          - 📄 **metadata.json** <-- session voice preset metadata
-          - 📂 **npz**
-      - 📂 **session_2**
-      - **...**
-
-## Add voice to system voice presets via command line
-It's recommended to manage voice presets via UI. However if you want to add voice to voice presets via command line. Run the script below:
+## (Advanced features) Speaker customization 
+You can add voice presets to WavJourney to customize the voice actors. Simply provide the voice id, the description and a sample wav file, and WavJourney will pick the voice automatically based on the audio script. Predefined system voice presets are in `data/voice_presets`.
+
+You can manage voice presets via UI. Specifically, if you want to add voice to voice presets. Run the script via command line below:
 ```bash
-python add_voice_preset.py --id "id" --desc "description" --wav-path path/to/wav --session-id session-id
+python add_voice_preset.py --id "id" --desc "description" --wav-path path/to/wav --session-id ''
 ```
-if `session-id` is set to '', then you are adding to system voice presets
+What makes for good voice prompt? See detailed instructions <a href="https://github.com/gitmylo/bark-voice-cloning-HuBERT-quantizer">here</a>. 
+## Hardware requirement
+- The VRAM of the GPU in the default configuration should be greater than 16 GB.
+- Operation system: Linux.
+
+## Citation
+If you find this work useful, you can cite the paper below:
+
+    @article{liu2023wavjourney,
+        title   = {WavJourney: Compositional Audio Creation with Large Language Models},
+        author  = {Liu, Xubo and Zhu, Zhongkai and Liu, Haohe and Yuan, Yi and Huang, Qiushi and Liang, Jinhua and Cao, Yin and Kong, Qiuqiang and Plumbley, Mark D and Wang, Wenwu},
+        journal = {arXiv preprint arXiv:2307.14335},
+        year    = {2023}
+    }
+
+[!["Buy Me A Coffee"](https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png)](https://www.buymeacoffee.com/liuxubo)
+
+## Appreciation
+- [Bark](https://github.com/suno-ai/bark) for a zero-shot text-to-speech synthesis model.
+- [AudioCraft](https://github.com/facebookresearch/audiocraft) for state-of-the-art audio generation models.
+
+## Disclaimer
+We are not responsible for audio generated using semantics created by this model. Just don't use it for illegal purposes.
+

VoiceFixer/app.py

@@ -1,55 +0,0 @@
-from genericpath import exists
-import os
-import os.path
-import logging
-from voicefixer import VoiceFixer
-from flask import Flask, request, jsonify
-
-# Configure the logging format and level
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(levelname)s - %(message)s'
-)
-
-# Create a FileHandler for the log file
-os.makedirs('services_logs', exist_ok=True)
-log_filename = 'services_logs/Speech-Restoration.log'
-file_handler = logging.FileHandler(log_filename, mode='w')
-file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
-
-# Add the FileHandler to the root logger
-logging.getLogger('').addHandler(file_handler)
-
-# Initialize the model here
-vf = VoiceFixer()
-logging.info('VoiceFixer is loaded ...')
-
-app = Flask(__name__)
-
-@app.route('/fix_audio', methods=['POST'])
-def fix_audio():
-    # Receive the text from the POST request
-    data = request.json
-    processfile = data['processfile']
-
-    logging.info(f'Fixing {processfile} ...')
-
-    try:
-        vf.restore(input=processfile, output=processfile, cuda=True, mode=0)
-        
-        # Return success message and the filename of the generated audio
-        return jsonify({'message': 'Speech restored successfully', 'file': processfile})
-
-    except Exception as e:
-        # Return error message if something goes wrong
-        return jsonify({'API error': str(e)}), 500
-    
-
-if __name__ == '__main__':
-    import yaml
-    with open('config.yaml', 'r') as file:
-        config = yaml.safe_load(file)
-
-    service_port = config['Speech-Restoration']['service-port']
-    app.run(debug=False, port=service_port)
-

VoiceParser/app.py

@@ -1,58 +0,0 @@
-from genericpath import exists
-import os
-import os.path
-import logging
-import yaml
-from model import VoiceParser
-from flask import Flask, request, jsonify
-
-with open('config.yaml', 'r') as file:
-    config = yaml.safe_load(file)
-
-service_port = config['Voice-Parser']['service-port']
-vp_device = config['Voice-Parser']['device']
-
-# Configure the logging format and level
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(levelname)s - %(message)s'
-)
-
-# Create a FileHandler for the log file
-os.makedirs('services_logs', exist_ok=True)
-log_filename = 'services_logs/Voice-Parser.log'
-file_handler = logging.FileHandler(log_filename, mode='w')
-file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
-
-# Add the FileHandler to the root logger
-logging.getLogger('').addHandler(file_handler)
-
-# Initialize the model here
-vp = VoiceParser(device=vp_device)
-logging.info('VoiceParser is loaded ...')
-
-app = Flask(__name__)
-
-@app.route('/parse_voice', methods=['POST'])
-def parse_voice():
-    # Receive the text from the POST request
-    data = request.json
-    wav_path = data['wav_path']
-    out_dir = data['out_dir']
-
-    logging.info(f'Parsing {wav_path} ...')
-
-    try:
-        vp.extract_acoustic_embed(wav_path, out_dir)
-        
-        # Return success message and the filename of the generated audio
-        return jsonify({'message': f'Sucessfully parsed {wav_path}'})
-
-    except Exception as e:
-        # Return error message if something goes wrong
-        return jsonify({'API error': str(e)}), 500
-    
-
-if __name__ == '__main__':
-    app.run(debug=False, port=service_port)
-

VoiceParser/model.py

@@ -8,9 +8,9 @@ torchaudio.set_audio_backend("soundfile")  # Use 'soundfile' backend
 
 from encodec import EncodecModel
 from encodec.utils import convert_audio
-from hubert_manager import HuBERTManager
-from pre_kmeans_hubert import CustomHubert
-from customtokenizer import CustomTokenizer
+from .hubert_manager import HuBERTManager
+from .pre_kmeans_hubert import CustomHubert
+from .customtokenizer import CustomTokenizer
 
 class VoiceParser():
     def __init__(self, device='cpu'):

pipeline.py

@@ -1,11 +1,8 @@
-import argparse
 import datetime
 import os
-import subprocess
 from string import Template
 import openai
 import re
-from pathlib import Path
 import glob
 from utils import get_key
 import pickle
@@ -13,7 +10,6 @@ import time
 import json5
 from retrying import retry
 from code_generator import check_json_script, collect_and_check_audio_data
-from tabulate import tabulate
 import random
 import string
 
@@ -21,7 +17,8 @@ import utils
 import voice_presets
 from code_generator import AudioCodeGenerator
 
-USE_OPENAI_CACHE = True
+# Enable this for debugging
+USE_OPENAI_CACHE = False
 openai_cache = []
 if USE_OPENAI_CACHE:
     os.makedirs('cache', exist_ok=True)
@@ -203,7 +200,7 @@ def generate_audio(session_id, json_script):
     voices = voice_presets.get_merged_voice_presets(session_id)
 
     # Step 2
-    json_script_to_char_voice_map(json_script, voices, output_path)
+    char_voice_map = json_script_to_char_voice_map(json_script, voices, output_path)
     # Step 3
     json_script_filename = output_path / 'audio_script.json'
     char_voice_map_filename = output_path / 'character_voice_map.json'
@@ -214,22 +211,9 @@ def generate_audio(session_id, json_script):
 
     result_wav_filename = output_audio_path / f'{result_wav_basename}.wav'
     print(f'Done all processes, result: {result_wav_filename}')
-    return result_wav_filename
+    return result_wav_filename, char_voice_map
 
 # Convenient function call used by wavjourney_cli
 def full_steps(session_id, input_text):
     json_script = generate_json_file(session_id, input_text)
     return generate_audio(session_id, json_script)
-
-def convert_json_to_md(audio_script_response):
-    audio_json_data = json5.loads(audio_script_response)
-    table = [[node.get(field, 'N/A') for field in ["audio_type", "layout", "id", "character", "action", 'vol']] +
-             [node.get("desc", "N/A") if node.get("audio_type") != "speech" else node.get("text", "N/A")] +
-             [node.get("len", "Auto") if "len" in node else "Auto"]
-             for i, node in enumerate(audio_json_data)]
-
-    headers = ["Audio Type", "Layout", "ID", "Character", "Action", 'Volume', "Description", "Length" ]
-
-    # Tabulate
-    table_txt = tabulate(table, headers, tablefmt="github")
-    return table_txt

scripts/EnvsSetup.sh

@@ -0,0 +1,7 @@
+conda env create -f Envs/WavJourney.yml && \
+conda env update -f Envs/Bark.yml && \
+conda env update -f Envs/AudioCraft.yml && \
+conda run --live-stream -n WavJourney pip install -U git+https://git@github.com/facebookresearch/audiocraft@c5157b5bf14bf83449c17ea1eeb66c19fb4bc7f0#egg=audiocraft && \
+conda run --live-stream -n WavJourney pip install -U --no-deps voicefixer==0.1.2 && \
+conda run --live-stream -n WavJourney pip install -U --no-deps numpy==1.21 && \
+conda run --live-stream -n WavJourney pip install -U --no-deps librosa==0.8.1

scripts/download_models.py

@@ -6,26 +6,24 @@ with open('config.yaml', 'r') as file:
     config = yaml.safe_load(file)
 
 # Extract values for each application
-tts_env = config['Text-to-Speech']['env']
+ttm_model_size = config['AudioCraft']['ttm_model_size']
+tta_model_size = config['AudioCraft']['tta_model_size']
 
-ttm_env = config['Text-to-Music']['env']
-ttm_model_size = config['Text-to-Music']['model_size']
-
-tta_env = config['Text-to-Audio']['env']
-
-sr_env = config['Speech-Restoration']['env']
 
 # Downloading the TTS models
 print('Step 1: Downloading TTS model ...')
-os.system(f'conda run --live-stream -n {tts_env} python -c \'from transformers import BarkModel; BarkModel.from_pretrained("suno/bark")\'')
+os.system(f'conda run --live-stream -n WavJourney python -c \'from transformers import BarkModel; BarkModel.from_pretrained("suno/bark")\'')
 
 print('Step 2: Downloading TTA model ...')
-os.system(f'conda run --live-stream -n {tta_env} python -c \'from audiocraft.models import AudioGen; tta_model = AudioGen.get_pretrained("facebook/audiogen-medium")\'')
+os.system(f'conda run --live-stream -n WavJourney python -c \'from audiocraft.models import AudioGen; tta_model = AudioGen.get_pretrained("facebook/audiogen-{tta_model_size}")\'')
 
 print('Step 3: Downloading TTM model ...')
-os.system(f'conda run --live-stream -n {ttm_env} python -c \'from audiocraft.models import MusicGen; tta_model = MusicGen.get_pretrained("facebook/musicgen-{ttm_model_size}")\'')
+os.system(f'conda run --live-stream -n WavJourney python -c \'from audiocraft.models import MusicGen; tta_model = MusicGen.get_pretrained("facebook/musicgen-{ttm_model_size}")\'')
 
 print('Step 4: Downloading SR model ...')
-os.system(f'conda run --live-stream -n {sr_env} python -c \'from voicefixer import VoiceFixer; vf = VoiceFixer()\'')
+os.system(f'conda run --live-stream -n WavJourney python -c \'from voicefixer import VoiceFixer; vf = VoiceFixer()\'')
+
+print('Step 5: Downloading VP model ...')
+os.system(f'conda run --live-stream -n WavJourney python -c \'from VoiceParser.model import VoiceParser; vp = VoiceParser(device="cpu")\'')
 
 print('All models successfully downloaded!')

scripts/kill_services.py

@@ -6,23 +6,11 @@ with open('config.yaml', 'r') as file:
     config = yaml.safe_load(file)
 
 # Extract values for each application
-tts_port = config['Text-to-Speech']['service-port']
-
-ttm_port = config['Text-to-Music']['service-port']
-
-tta_port = config['Text-to-Audio']['service-port']
-
-sr_port = config['Speech-Restoration']['service-port']
-
-vp_port = config['Voice-Parser']['service-port']
-
+service_port = config['Service-Port']
 
 # Execute the commands 
-os.system(f'kill $(lsof -t -i :{tts_port})')
-os.system(f'kill $(lsof -t -i :{tta_port})')
-os.system(f'kill $(lsof -t -i :{ttm_port})')
-os.system(f'kill $(lsof -t -i :{sr_port})')
-os.system(f'kill $(lsof -t -i :{vp_port})')
+os.system(f'kill $(lsof -t -i :{service_port})')
+
 
 
 

scripts/restart_services.sh

@@ -1,2 +0,0 @@
-python scripts/kill_services.py
-python scripts/start_services.py

scripts/start_service_and_ui.sh

@@ -0,0 +1,2 @@
+nohup conda run --live-stream -n WavJourney python services.py > services_logs/service.out 2>&1 &
+conda run --live-stream -n WavJourney python -u ui_client.py 2>&1 | stdbuf -oL tee services_logs/wavejourney.out

scripts/start_services.py

@@ -1,41 +0,0 @@
-import yaml
-import os
-
-# Read the YAML file
-with open('config.yaml', 'r') as file:
-    config = yaml.safe_load(file)
-
-os.makedirs('services_logs', exist_ok=True)
-
-# Extract values for each application
-tts_model = config['Text-to-Speech']['model']
-tts_env = config['Text-to-Speech']['env']
-
-ttm_model = config['Text-to-Music']['model']
-ttm_env = config['Text-to-Music']['env']
-
-tta_model = config['Text-to-Audio']['model']
-tta_env = config['Text-to-Audio']['env']
-
-sr_model = config['Speech-Restoration']['model']
-sr_env = config['Speech-Restoration']['env']
-enable_sr = config['Speech-Restoration']['Enable']
-
-vp_model = config['Voice-Parser']['model']
-vp_env = config['Voice-Parser']['env']
-
-# Execute the commands 
-os.system(f'nohup conda run --live-stream -n {tts_env} python {tts_model}/app.py > services_logs/meta_tts.out 2>&1 &')
-os.system(f'nohup conda run --live-stream -n {vp_env} python {vp_model}/app.py > services_logs/meta_vp.out 2>&1 &')
-
-if enable_sr:
-    os.system(f'nohup conda run --live-stream -n {sr_env} python {sr_model}/app.py > services_logs/meta_sr.out 2>&1 &')
-
-# Using AudioCraft for TTA & TTM
-if tta_env == ttm_env:
-    os.system(f'nohup conda run --live-stream -n {ttm_env} python {ttm_model}/app.py > services_logs/meta_tta_ttm.out 2>&1 &')
-
-# Using AudioLDM for TTA, MusicGen for TTM
-if tta_env != ttm_env:
-    os.system(f'nohup conda run --live-stream -n {tta_env} python {tta_model}/app.py > services_logs/meta_tta.out 2>&1 &')
-    os.system(f'nohup conda run --live-stream -n {ttm_env} python {ttm_model}/app.py > services_logs/meta_ttm.out 2>&1 &')

scripts/start_services.sh

@@ -0,0 +1 @@
+nohup conda run --live-stream -n WavJourney python services.py > services_logs/service.out 2>&1 &

scripts/start_ui.sh

@@ -1 +1 @@
-conda run --live-stream -n WavJourney gradio ui_client.py
+conda run --live-stream -n WavJourney python -u ui_client.py 2>&1 | stdbuf -oL tee services_logs/wavejourney.out

Bark/app.py → services.py

@@ -1,17 +1,13 @@
 import os
-import sys
-sys.path.append('../AudioJourney')
-import logging
 import yaml
-import numpy as np
+import logging
+import nltk
 import torch
 import torchaudio
 from torchaudio.transforms import SpeedPerturbation
-import nltk
 from APIs import WRITE_AUDIO, LOUDNESS_NORM
+from utils import fade
 from flask import Flask, request, jsonify
-from transformers import BarkModel, AutoProcessor
-
 
 with open('config.yaml', 'r') as file:
     config = yaml.safe_load(file)
@@ -24,32 +20,119 @@ logging.basicConfig(
 
 # Create a FileHandler for the log file
 os.makedirs('services_logs', exist_ok=True)
-log_filename = 'services_logs/Text-to-Speech.log'
+log_filename = 'services_logs/Wav-API.log'
 file_handler = logging.FileHandler(log_filename, mode='w')
 file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
 
 # Add the FileHandler to the root logger
 logging.getLogger('').addHandler(file_handler)
 
-# Initialize the model here
+
+"""
+Initialize the AudioCraft models here
+"""
+from audiocraft.models import AudioGen, MusicGen
+tta_model_size = config['AudioCraft']['tta_model_size']
+tta_model = AudioGen.get_pretrained(f'facebook/audiogen-{tta_model_size}')
+logging.info(f'AudioGen ({tta_model_size}) is loaded ...')
+
+ttm_model_size = config['AudioCraft']['ttm_model_size']
+ttm_model = MusicGen.get_pretrained(f'facebook/musicgen-{ttm_model_size}')
+logging.info(f'MusicGen ({ttm_model_size}) is loaded ...')
+
+
+"""
+Initialize the BarkModel here
+"""
+from transformers import BarkModel, AutoProcessor
 SPEED = float(config['Text-to-Speech']['speed'])
 speed_perturb = SpeedPerturbation(32000, [SPEED])
-
-logging.info('Loading Bark model ...')
-# TODO: fp16?
-model = BarkModel.from_pretrained("suno/bark")
+tts_model = BarkModel.from_pretrained("suno/bark")
 device = "cuda:0" if torch.cuda.is_available() else "cpu"
-model = model.to(device)
-model = model.to_bettertransformer()    # Flash attention
-SAMPLE_RATE = model.generation_config.sample_rate
+tts_model = tts_model.to(device)
+tts_model = tts_model.to_bettertransformer()    # Flash attention
+SAMPLE_RATE = tts_model.generation_config.sample_rate
 SEMANTIC_TEMPERATURE = 0.9
 COARSE_TEMPERATURE = 0.5
 FINE_TEMPERATURE = 0.5
-
 processor = AutoProcessor.from_pretrained("suno/bark")
+logging.info('Bark model is loaded ...')
+
+
+""" 
+Initialize the VoiceFixer model here
+"""
+from voicefixer import VoiceFixer
+vf = VoiceFixer()
+logging.info('VoiceFixer is loaded ...')
+
+
+"""
+Initalize the VoiceParser model here
+"""
+from VoiceParser.model import VoiceParser
+vp_device = config['Voice-Parser']['device']
+vp = VoiceParser(device=vp_device)
+logging.info('VoiceParser is loaded ...')
+
 
 app = Flask(__name__)
 
+
+@app.route('/generate_audio', methods=['POST'])
+def generate_audio():
+    # Receive the text from the POST request
+    data = request.json
+    text = data['text']
+    length = float(data.get('length', 5.0))
+    volume = float(data.get('volume', -35))
+    output_wav = data.get('output_wav', 'out.wav')
+
+    logging.info(f'TTA (AudioGen): Prompt: {text}, length: {length} seconds, volume: {volume} dB')
+    
+    try:
+        tta_model.set_generation_params(duration=length)  
+        wav = tta_model.generate([text])  
+        wav = torchaudio.functional.resample(wav, orig_freq=16000, new_freq=32000)
+
+        wav = wav.squeeze().cpu().detach().numpy()
+        wav = fade(LOUDNESS_NORM(wav, volumn=volume))
+        WRITE_AUDIO(wav, name=output_wav)
+
+        # Return success message and the filename of the generated audio
+        return jsonify({'message': f'Text-to-Audio generated successfully | {text}', 'file': output_wav})
+
+    except Exception as e:
+        return jsonify({'API error': str(e)}), 500
+
+
+@app.route('/generate_music', methods=['POST'])
+def generate_music():
+    # Receive the text from the POST request
+    data = request.json
+    text = data['text']
+    length = float(data.get('length', 5.0))
+    volume = float(data.get('volume', -35))
+    output_wav = data.get('output_wav', 'out.wav')
+
+    logging.info(f'TTM (MusicGen): Prompt: {text}, length: {length} seconds, volume: {volume} dB')
+
+
+    try:
+        ttm_model.set_generation_params(duration=length)  
+        wav = ttm_model.generate([text])  
+        wav = wav[0][0].cpu().detach().numpy()
+        wav = fade(LOUDNESS_NORM(wav, volumn=volume))
+        WRITE_AUDIO(wav, name=output_wav)
+
+        # Return success message and the filename of the generated audio
+        return jsonify({'message': f'Text-to-Music generated successfully | {text}', 'file': output_wav})
+
+    except Exception as e:
+        # Return error message if something goes wrong
+        return jsonify({'API error': str(e)}), 500
+
+
 @app.route('/generate_speech', methods=['POST'])
 def generate_speech():
     # Receive the text from the POST request
@@ -77,7 +160,7 @@ def generate_speech():
 
             with torch.inference_mode():
                 # TODO: min_eos_p?
-                output = model.generate(
+                output = tts_model.generate(
                     **inputs,
                     do_sample = True,
                     semantic_temperature = SEMANTIC_TEMPERATURE,
@@ -99,11 +182,49 @@ def generate_speech():
         return jsonify({'message': f'Text-to-Speech generated successfully | {speaker_id}: {text}', 'file': output_wav})
 
     except Exception as e:
-        raise e
+        # Return error message if something goes wrong
+        return jsonify({'API error': str(e)}), 500
+
+
+@app.route('/fix_audio', methods=['POST'])
+def fix_audio():
+    # Receive the text from the POST request
+    data = request.json
+    processfile = data['processfile']
+
+    logging.info(f'Fixing {processfile} ...')
+
+    try:
+        vf.restore(input=processfile, output=processfile, cuda=True, mode=0)
+        
+        # Return success message and the filename of the generated audio
+        return jsonify({'message': 'Speech restored successfully', 'file': processfile})
+
+    except Exception as e:
+        # Return error message if something goes wrong
+        return jsonify({'API error': str(e)}), 500
+
+
+@app.route('/parse_voice', methods=['POST'])
+def parse_voice():
+    # Receive the text from the POST request
+    data = request.json
+    wav_path = data['wav_path']
+    out_dir = data['out_dir']
+
+    logging.info(f'Parsing {wav_path} ...')
+
+    try:
+        vp.extract_acoustic_embed(wav_path, out_dir)
+        
+        # Return success message and the filename of the generated audio
+        return jsonify({'message': f'Sucessfully parsed {wav_path}'})
+
+    except Exception as e:
         # Return error message if something goes wrong
         return jsonify({'API error': str(e)}), 500
 
 
 if __name__ == '__main__':
-    service_port = config['Text-to-Speech']['service-port']
+    service_port = config['Service-Port']
     app.run(debug=False, port=service_port)

share_btn.py

@@ -0,0 +1,74 @@
+community_icon_html = """<svg id="share-btn-share-icon" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32">
+    <path d="M20.6081 3C21.7684 3 22.8053 3.49196 23.5284 4.38415C23.9756 4.93678 24.4428 5.82749 24.4808 7.16133C24.9674 7.01707 25.4353 6.93643 25.8725 6.93643C26.9833 6.93643 27.9865 7.37587 28.696 8.17411C29.6075 9.19872 30.0124 10.4579 29.8361 11.7177C29.7523 12.3177 29.5581 12.8555 29.2678 13.3534C29.8798 13.8646 30.3306 14.5763 30.5485 15.4322C30.719 16.1032 30.8939 17.5006 29.9808 18.9403C30.0389 19.0342 30.0934 19.1319 30.1442 19.2318C30.6932 20.3074 30.7283 21.5229 30.2439 22.6548C29.5093 24.3704 27.6841 25.7219 24.1397 27.1727C21.9347 28.0753 19.9174 28.6523 19.8994 28.6575C16.9842 29.4379 14.3477 29.8345 12.0653 29.8345C7.87017 29.8345 4.8668 28.508 3.13831 25.8921C0.356375 21.6797 0.754104 17.8269 4.35369 14.1131C6.34591 12.058 7.67023 9.02782 7.94613 8.36275C8.50224 6.39343 9.97271 4.20438 12.4172 4.20438H12.4179C12.6236 4.20438 12.8314 4.2214 13.0364 4.25468C14.107 4.42854 15.0428 5.06476 15.7115 6.02205C16.4331 5.09583 17.134 4.359 17.7682 3.94323C18.7242 3.31737 19.6794 3 20.6081 3ZM20.6081 5.95917C20.2427 5.95917 19.7963 6.1197 19.3039 6.44225C17.7754 7.44319 14.8258 12.6772 13.7458 14.7131C13.3839 15.3952 12.7655 15.6837 12.2086 15.6837C11.1036 15.6837 10.2408 14.5497 12.1076 13.1085C14.9146 10.9402 13.9299 7.39584 12.5898 7.1776C12.5311 7.16799 12.4731 7.16355 12.4172 7.16355C11.1989 7.16355 10.6615 9.33114 10.6615 9.33114C10.6615 9.33114 9.0863 13.4148 6.38031 16.206C3.67434 18.998 3.5346 21.2388 5.50675 24.2246C6.85185 26.2606 9.42666 26.8753 12.0653 26.8753C14.8021 26.8753 17.6077 26.2139 19.1799 25.793C19.2574 25.7723 28.8193 22.984 27.6081 20.6107C27.4046 20.212 27.0693 20.0522 26.6471 20.0522C24.9416 20.0522 21.8393 22.6726 20.5057 22.6726C20.2076 22.6726 19.9976 22.5416 19.9116 22.222C19.3433 20.1173 28.552 19.2325 27.7758 16.1839C27.639 15.6445 27.2677 15.4256 26.746 15.4263C24.4923 15.4263 19.4358 19.5181 18.3759 19.5181C18.2949 19.5181 18.2368 19.4937 18.2053 19.4419C17.6743 18.557 17.9653 17.9394 21.7082 15.6009C25.4511 13.2617 28.0783 11.8545 26.5841 10.1752C26.4121 9.98141 26.1684 9.8956 25.8725 9.8956C23.6001 9.89634 18.2311 14.9403 18.2311 14.9403C18.2311 14.9403 16.7821 16.496 15.9057 16.496C15.7043 16.496 15.533 16.4139 15.4169 16.2112C14.7956 15.1296 21.1879 10.1286 21.5484 8.06535C21.7928 6.66715 21.3771 5.95917 20.6081 5.95917Z" fill="#FF9D00"></path>
+    <path d="M5.50686 24.2246C3.53472 21.2387 3.67446 18.9979 6.38043 16.206C9.08641 13.4147 10.6615 9.33111 10.6615 9.33111C10.6615 9.33111 11.2499 6.95933 12.59 7.17757C13.93 7.39581 14.9139 10.9401 12.1069 13.1084C9.29997 15.276 12.6659 16.7489 13.7459 14.713C14.8258 12.6772 17.7747 7.44316 19.304 6.44221C20.8326 5.44128 21.9089 6.00204 21.5484 8.06532C21.188 10.1286 14.795 15.1295 15.4171 16.2118C16.0391 17.2934 18.2312 14.9402 18.2312 14.9402C18.2312 14.9402 25.0907 8.49588 26.5842 10.1752C28.0776 11.8545 25.4512 13.2616 21.7082 15.6008C17.9646 17.9393 17.6744 18.557 18.2054 19.4418C18.7372 20.3266 26.9998 13.1351 27.7759 16.1838C28.5513 19.2324 19.3434 20.1173 19.9117 22.2219C20.48 24.3274 26.3979 18.2382 27.6082 20.6107C28.8193 22.9839 19.2574 25.7722 19.18 25.7929C16.0914 26.62 8.24723 28.3726 5.50686 24.2246Z" fill="#FFD21E"></path>
+</svg>"""
+
+loading_icon_html = """<svg id="share-btn-loading-icon" style="display:none;" class="animate-spin"
+   style="color: #ffffff; 
+"
+   xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" fill="none" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><circle style="opacity: 0.25;" cx="12" cy="12" r="10" stroke="white" stroke-width="4"></circle><path style="opacity: 0.75;" fill="white" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path></svg>"""
+
+share_js = """async () => {
+	async function uploadFile(file){
+		const UPLOAD_URL = 'https://huggingface.co/uploads';
+		const response = await fetch(UPLOAD_URL, {
+			method: 'POST',
+			headers: {
+				'Content-Type': file.type,
+				'X-Requested-With': 'XMLHttpRequest',
+			},
+			body: file, /// <- File inherits from Blob
+		});
+		const url = await response.text();
+		return url;
+	}
+    async function getInputVideoFile(videoEl){
+        const res = await fetch(videoEl.src);
+        const blob = await res.blob();
+        const videoId = Date.now() % 200;
+        const fileName = `sd-perception-${{videoId}}.mp4`;
+        return new File([blob], fileName, { type: 'video/mp4' }); 
+	}
+    
+    async function audioToBase64(audioFile) {
+        return new Promise((resolve, reject) => {
+            let reader = new FileReader();
+            reader.readAsDataURL(audioFile);
+            reader.onload = () => resolve(reader.result);
+            reader.onerror = error => reject(error);
+          
+        });
+      }
+    const gradioEl = document.querySelector("gradio-app").shadowRoot || document.querySelector('body > gradio-app');
+    const inputPromptEl = gradioEl.querySelector('#prompt-in input').value;
+    const outputVideoEl = gradioEl.querySelector('#output-video video');
+    
+    let titleTxt = `WavJourney: ${inputPromptEl}`;
+
+    const shareBtnEl = gradioEl.querySelector('#share-btn');
+    const shareIconEl = gradioEl.querySelector('#share-btn-share-icon');
+    const loadingIconEl = gradioEl.querySelector('#share-btn-loading-icon');
+    if(!outputVideoEl){
+        return;
+    };
+    shareBtnEl.style.pointerEvents = 'none';
+    shareIconEl.style.display = 'none';
+    loadingIconEl.style.removeProperty('display');
+    const outputVideo = await getInputVideoFile(outputVideoEl);
+    const urlOutputVideo = await uploadFile(outputVideo);
+    
+    const descriptionMd = `
+##### ${inputPromptEl}
+    
+${urlOutputVideo}
+`;
+    const params = new URLSearchParams({
+        title: titleTxt,
+        description: descriptionMd,
+    });
+	const paramsStr = params.toString();
+	window.open(`https://huggingface.co/spaces/Audio-AGI/WavJourney/discussions/new?${paramsStr}`, '_blank');
+    shareBtnEl.style.removeProperty('pointer-events');
+    shareIconEl.style.removeProperty('display');
+    loadingIconEl.style.display = 'none';
+}"""

ui_client.py

@@ -1,30 +1,63 @@
-import pdb
 import shutil
+import json5
 
+import openai
 import gradio as gr
+from tabulate import tabulate
 
-import pipeline
 import utils
+import pipeline
 from pipeline import generate_json_file, generate_audio
 from voice_presets import load_voice_presets_metadata, add_session_voice_preset, \
     remove_session_voice_preset
+from share_btn import community_icon_html, loading_icon_html, share_js
+
 
-import openai
 
 VOICE_PRESETS_HEADERS = ['ID', 'Description']
 DELETE_FILE_WHEN_DO_CLEAR = False
 DEBUG = False
 
 
+def convert_json_to_md(audio_script_response):
+    audio_json_data = json5.loads(audio_script_response)
+    table = [[node.get(field, 'N/A') for field in ["audio_type", "layout", "id", "character", "action", 'vol']] +
+             [node.get("desc", "N/A") if node.get("audio_type") != "speech" else node.get("text", "N/A")] +
+             [node.get("len", "Auto") if "len" in node else "Auto"]
+             for i, node in enumerate(audio_json_data)]
+
+    headers = ["Audio Type", "Layout", "ID", "Character", "Action", 'Volume', "Description", "Length" ]
+
+    # Tabulate
+    table_txt = tabulate(table, headers, tablefmt="github")
+    return table_txt
+
+
+def convert_char_voice_map_to_md(char_voice_map):
+    table =[[character, char_voice_map[character]["id"]] for character in char_voice_map]
+    headers = ["Character", "Voice"]
+    # Tabulate
+    table_txt = tabulate(table, headers, tablefmt="github")
+    return table_txt
+
+
 def generate_script_fn(instruction, _state: gr.State):
     try:
         session_id = _state['session_id']
         json_script = generate_json_file(session_id, instruction)
-        table_text = pipeline.convert_json_to_md(json_script)
+        table_text = convert_json_to_md(json_script)
     except Exception as e:
         gr.Warning(str(e))
         print(f"Generating script error: {str(e)}")
-        return [None, gr.Button.update(interactive=False), _state, gr.Button.update(interactive=True)]
+        return [
+            None,
+            _state,
+            gr.Button.update(interactive=False),
+            gr.Button.update(interactive=True),
+            gr.Button.update(interactive=False),
+            gr.Button.update(interactive=False),
+        ]
+
     _state = {
         **_state,
         'session_id': session_id,
@@ -43,8 +76,11 @@ def generate_script_fn(instruction, _state: gr.State):
 def generate_audio_fn(state):
     btn_state = gr.Button.update(interactive=True)
     try:
-        audio_path = generate_audio(**state)
+        audio_path, char_voice_map = generate_audio(**state)
+        table_text = convert_char_voice_map_to_md(char_voice_map)
+        # TODO: output char_voice_map to a table
         return [
+            table_text,
             gr.make_waveform(str(audio_path)),
             btn_state,
             btn_state,
@@ -54,7 +90,11 @@ def generate_audio_fn(state):
     except Exception as e:
         print(f"Generation audio error: {str(e)}")
         gr.Warning(str(e))
+        # For debugging, uncomment the line below
+        #raise e
+
     return [
+        None,
         None,
         btn_state,
         btn_state,
@@ -164,40 +204,262 @@ def add_voice_preset(vp_id, vp_desc, file, ui_state, added_voice_preset):
             df_visible, del_visible]
 
 
-with gr.Blocks() as interface:
+css = """
+        a {
+            color: inherit;
+            text-decoration: underline;
+        }
+        .gradio-container {
+            font-family: 'IBM Plex Sans', sans-serif;
+        }
+        .gr-button {
+            color: white;
+            border-color: #000000;
+            background: #000000;
+        }
+        input[type='range'] {
+            accent-color: #000000;
+        }
+        .dark input[type='range'] {
+            accent-color: #dfdfdf;
+        }
+        .container {
+            max-width: 730px;
+            margin: auto;
+            padding-top: 1.5rem;
+        }
+        #gallery {
+            min-height: 22rem;
+            margin-bottom: 15px;
+            margin-left: auto;
+            margin-right: auto;
+            border-bottom-right-radius: .5rem !important;
+            border-bottom-left-radius: .5rem !important;
+        }
+        #gallery>div>.h-full {
+            min-height: 20rem;
+        }
+        .details:hover {
+            text-decoration: underline;
+        }
+        .gr-button {
+            white-space: nowrap;
+        }
+        .gr-button:focus {
+            border-color: rgb(147 197 253 / var(--tw-border-opacity));
+            outline: none;
+            box-shadow: var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow, 0 0 #0000);
+            --tw-border-opacity: 1;
+            --tw-ring-offset-shadow: var(--tw-ring-inset) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color);
+            --tw-ring-shadow: var(--tw-ring-inset) 0 0 0 calc(3px var(--tw-ring-offset-width)) var(--tw-ring-color);
+            --tw-ring-color: rgb(191 219 254 / var(--tw-ring-opacity));
+            --tw-ring-opacity: .5;
+        }
+        #advanced-btn {
+            font-size: .7rem !important;
+            line-height: 19px;
+            margin-top: 12px;
+            margin-bottom: 12px;
+            padding: 2px 8px;
+            border-radius: 14px !important;
+        }
+        #advanced-options {
+            margin-bottom: 20px;
+        }
+        .footer {
+            margin-bottom: 45px;
+            margin-top: 35px;
+            text-align: center;
+            border-bottom: 1px solid #e5e5e5;
+        }
+        .footer>p {
+            font-size: .8rem;
+            display: inline-block;
+            padding: 0 10px;
+            transform: translateY(10px);
+            background: white;
+        }
+        .dark .footer {
+            border-color: #303030;
+        }
+        .dark .footer>p {
+            background: #0b0f19;
+        }
+        .acknowledgments h4{
+            margin: 1.25em 0 .25em 0;
+            font-weight: bold;
+            font-size: 115%;
+        }
+        #container-advanced-btns{
+            display: flex;
+            flex-wrap: wrap;
+            justify-content: space-between;
+            align-items: center;
+        }
+        .animate-spin {
+            animation: spin 1s linear infinite;
+        }
+        @keyframes spin {
+            from {
+                transform: rotate(0deg);
+            }
+            to {
+                transform: rotate(360deg);
+            }
+        }
+        #share-btn-container {
+            display: flex; padding-left: 0.5rem !important; padding-right: 0.5rem !important; background-color: #000000; justify-content: center; align-items: center; border-radius: 9999px !important; width: 13rem;
+            margin-top: 10px;
+            margin-left: auto;
+        }
+        #share-btn {
+            all: initial; color: #ffffff;font-weight: 600; cursor:pointer; font-family: 'IBM Plex Sans', sans-serif; margin-left: 0.5rem !important; padding-top: 0.25rem !important; padding-bottom: 0.25rem !important;right:0;
+        }
+        #share-btn * {
+            all: unset;
+        }
+        #share-btn-container div:nth-child(-n+2){
+            width: auto !important;
+            min-height: 0px !important;
+        }
+        #share-btn-container .wrap {
+            display: none !important;
+        }
+        .gr-form{
+            flex: 1 1 50%; border-top-right-radius: 0; border-bottom-right-radius: 0;
+        }
+        #prompt-container{
+            gap: 0;
+        }
+        #generated_id{
+            min-height: 700px
+        }
+        #setting_id{
+          margin-bottom: 12px;
+          text-align: center;
+          font-weight: 900;
+        }
+"""
+
+with gr.Blocks(css=css) as interface:
+
+    gr.HTML(
+            """
+                <div style="text-align: center; max-width: 700px; margin: 0 auto;">
+                <div
+                    style="
+                    display: inline-flex;
+                    align-items: center;
+                    gap: 0.8rem;
+                    font-size: 1.75rem;
+                    "
+                >
+                    <h1 style="font-weight: 900; margin-bottom: 7px; line-height: normal;">
+                    WavJourney: Compositional Audio Creation with LLMs
+                    </h1>
+                </div>
+                <p style="margin-bottom: 10px; margin-top: 10px; font-size: 94%">
+                    <a href="https://arxiv.org/abs/2307.14335">[Paper]</a> <a href="https://audio-agi.github.io/WavJourney_demopage/">[Demo Page]</a> <a href="https://github.com/Audio-AGI/WavJourney">[GitHub]</a> <a href="https://discord.com/invite/5Hqu9NmA8V">[Join Discord]</a>
+                </p>
+                </div>
+            """
+        )
+
+    gr.HTML(
+        """
+        <p>For faster inference without waiting in queue, you may duplicate the space and upgrade to GPU (VRAM>16G) in settings.
+        <br>
+        <a href="https://huggingface.co/spaces/Audio-AGI/WavJourney?duplicate=true">
+        <img style="margin-top: 0em; margin-bottom: 0em" src="https://bit.ly/3gLdBN6" alt="Duplicate Space"></a>
+        <p/>
+    """
+    )
+
+
+
     system_voice_presets = get_system_voice_presets()
     # State
     ui_state = gr.State(value={'session_id': pipeline.init_session()})
     selected_voice_presets = gr.State(value={'selected_voice_preset': None})
     added_voice_preset_state = gr.State(value={'added_file': None, 'count': 0})
     # UI Component
-    key_text_input = gr.Textbox(label='Please Enter OPENAI Key for acessing GPT4', lines=1, placeholder="Input instruction here.",
-                            value='')
-    text_input_value = '' if DEBUG is False else "News channel BBC broadcast about Trump playing street fighter 6 against Biden"
-    text_input = gr.Textbox(label='Input', lines=2, placeholder="Input instruction here.",
-                            value=text_input_value)
-    markdown_output = gr.Markdown(label='Audio Script', lines=2)
+    gr.Markdown(
+    """
+    How can I access GPT-4? <a href="https://platform.openai.com/account/api-keys">[Guidence1]</a><a href="https://help.openai.com/en/articles/7102672-how-can-i-access-gpt-4">[Guidence2]</a>
+    """
+    )
+    key_text_input = gr.Textbox(label='Please Enter OPENAI Key for accessing GPT-4 API', lines=1, placeholder="OPENAI Key here.",
+                            value=utils.get_key())
+    text_input_value = '' if DEBUG is False else "Generate a one-minute introduction to quantum mechanics"
+    
+    text_input = gr.Textbox(
+        label='Input Text Instruction', 
+        lines=2, 
+        placeholder="Input instruction here (e.g., Generate a one-minute introduction to quantum mechanics).",
+        value=text_input_value,
+        elem_id="prompt-in",)
+
+    gr.Markdown(
+    """
+    Clicking 'Generate Script' button, the generated audio script will be displayed below.
+    """
+    )
+    audio_script_markdown = gr.Markdown(label='Audio Script')
     generate_script_btn = gr.Button(value='Generate Script', interactive=False)
-    audio_output = gr.Video(type='filepath')
+    
+    gr.Markdown(
+    """
+    Clicking 'Generate Audio' button, the voice mapping results & generated audio will be displayed below.
+    """
+    )
+    char_voice_map_markdown = gr.Markdown(label='Character-to-voice Map')
+
+    audio_output = gr.Video(elem_id="output-video")
+
     generate_audio_btn = gr.Button(value='Generate Audio', interactive=False)
-    clear_btn = gr.ClearButton(value='Clear Inputs')
+
+    clear_btn = gr.ClearButton(value='Clear All')
+    
+    # share to community
+    with gr.Group(elem_id="share-btn-container", visible=False):
+        community_icon = gr.HTML(community_icon_html)
+        loading_icon = gr.HTML(loading_icon_html)
+        share_button = gr.Button(value="Share to community", elem_id="share-btn")
+
     # System Voice Presets
     gr.Markdown(label='System Voice Presets', value='# System Voice Presets')
     system_markdown_voice_presets = gr.Dataframe(label='System Voice Presets', headers=VOICE_PRESETS_HEADERS,
                                                  value=system_voice_presets)
     # User Voice Preset Related
-    gr.Markdown(label='User Voice Presets', value='# User Voice Presets')
-    get_voice_preset_to_list(ui_state)
-    voice_presets_df = gr.Dataframe(headers=VOICE_PRESETS_HEADERS, col_count=len(VOICE_PRESETS_HEADERS),
+    gr.Markdown('# (Optional) Speaker Customization ')
+    with gr.Accordion("Click to add speakers", open=False):
+        gr.Markdown(label='User Voice Presets', value='## User Voice Presets')
+        get_voice_preset_to_list(ui_state)
+        voice_presets_df = gr.Dataframe(headers=VOICE_PRESETS_HEADERS, col_count=len(VOICE_PRESETS_HEADERS),
                                     value=get_voice_preset_to_list(ui_state), interactive=False, visible=False)
     # voice_presets_ds = gr.Dataset(components=[gr.Dataframe(visible=True)], samples=get_voice_preset_to_list(ui_state))
-    del_voice_btn = gr.Button(value='Delete Selected Voice Preset', visible=False)
-    gr.Markdown(label='Add Voice Preset', value='## Add Voice Preset')
-    vp_text_id = gr.Textbox(label='Id', lines=1, placeholder="Input voice preset id here.")
-    vp_text_desc = gr.Textbox(label='Desc', lines=1, placeholder="Input description here.")
-    vp_file = gr.File(label='Wav File', type='file', description='Upload your wav file here.', file_types=['.wav'],
-                      interactive=True)
-    vp_submit = gr.Button(label='Upload Voice Preset', value="Upload Voice Preset")
+        del_voice_btn = gr.Button(value='Delete Selected Voice Preset', visible=False)
+        gr.Markdown(label='Add Voice Preset', value='## Add Voice Preset')
+        gr.Markdown(
+        """
+        
+        What makes for good voice prompt? See detailed instructions <a href="https://github.com/gitmylo/bark-voice-cloning-HuBERT-quantizer">here</a>. 
+        """
+        )
+        vp_text_id = gr.Textbox(label='Id', lines=1, placeholder="Input voice preset id here.")
+        vp_text_desc = gr.Textbox(label='Desc', lines=1, placeholder="Input description here.")
+        vp_file = gr.File(label='Wav File', type='file', file_types=['.wav'],
+                        interactive=True)
+        vp_submit = gr.Button(label='Upload Voice Preset', value="Upload Voice Preset")
+
+    # disclaimer
+    gr.Markdown(
+        """
+    # Disclaimer
+    We are not responsible for audio generated using semantics created by WavJourney. Just don't use it for illegal purposes.
+    """
+    )
+
     # events
     key_text_input.change(fn=set_openai_key, inputs=[key_text_input], outputs=[key_text_input])
     text_input.change(fn=textbox_listener, inputs=[text_input], outputs=[generate_script_btn])
@@ -205,6 +467,7 @@ with gr.Blocks() as interface:
         fn=generate_audio_fn,
         inputs=[ui_state],
         outputs=[
+            char_voice_map_markdown,
             audio_output,
             generate_audio_btn,
             generate_script_btn,
@@ -214,7 +477,7 @@ with gr.Blocks() as interface:
         api_name='audio_journey',
     )
     generate_audio_btn.click(
-        fn=lambda _: [
+        fn=lambda: [
             gr.Button.update(interactive=False),
             gr.Button.update(interactive=False),
             gr.Button.update(interactive=False),
@@ -228,13 +491,13 @@ with gr.Blocks() as interface:
         ]
     )
     clear_btn.click(fn=clear_fn, inputs=ui_state,
-                    outputs=[text_input, audio_output, markdown_output, generate_audio_btn, generate_script_btn,
+                    outputs=[text_input, audio_output, audio_script_markdown, generate_audio_btn, generate_script_btn,
                              ui_state, voice_presets_df, del_voice_btn,
                              vp_text_id, vp_text_desc, vp_file])
     generate_script_btn.click(
         fn=generate_script_fn, inputs=[text_input, ui_state],
         outputs=[
-            markdown_output,
+            audio_script_markdown,
             ui_state,
             generate_audio_btn,
             generate_script_btn,
@@ -243,7 +506,7 @@ with gr.Blocks() as interface:
         ]
     )
     generate_script_btn.click(
-        fn=lambda _: [
+        fn=lambda: [
             gr.Button.update(interactive=False),
             gr.Button.update(interactive=False),
             gr.Button.update(interactive=False),
@@ -266,6 +529,10 @@ with gr.Blocks() as interface:
                              vp_submit,
                              voice_presets_df, del_voice_btn])
     vp_submit.click(lambda _: gr.Button.update(interactive=False), inputs=[vp_submit])
+    
+    # share to HF community
+    share_button.click(None, [], [], _js=share_js)
+
     # debug only
     # print_state_btn = gr.Button(value='Print State')
     # print_state_btn.click(fn=lambda state, state2: print(state, state2), inputs=[ui_state, selected_voice_presets])

webapp/app.prompt

@@ -1,18 +0,0 @@
-write a web app in python and flask and bootstrap.
-
-The UI:
-- input textbox named "InputTextbox" on top
-- "Generate All" button named GenerateAllButton at the same row as input textbox 1
-- a button "Text -> Script" called TextToScriptButton
-- A big textbox named "ScriptTextbox". The textbox should be set to wrap-word mode
-- A split line
-- a button "Script -> HAML" called ScriptToHAMLButton
-- A big textbox named "HAMLTextbox". The textbox should be set to wrap-word mode and display text with HTML syntax format.
-- A split line
-- a button "HAML -> Python Code" called HAMLToPythonCodeButton
-- A big textbox named "PythonCodeTextbox". The textbox should be set to wrap-word mode and display text with python syntax format.
-
-Behaviors:
-- When the user click TextToScriptButton, it will call ChatGPT API, which concat a prompt which is red from "prompts/text_to_audio_script.prompt" with the content from InputTextbox, and send it to ChatGPT, and output ChatGPT's response to ScriptTextbox.
-- When the user click ScriptToHAMLButton, it will call ChatGPT API, which concat a prompt which is red from "prompts/audio_script_to_HAML.prompt" with the content from ScriptTextbox, and send it to ChatGPT, and output ChatGPT's response to HAMLTextbox.
-- When the user click HAMLToPythonCodeButton, it will get the content of HAMLTextbox and pipe the content to the python script convert_haml_to_py_code.py, and return the script's output to PythonCodeTextbox.

webapp/app.py

@@ -1,43 +0,0 @@
-from flask import Flask, request, render_template
-import os
-import subprocess
-
-app = Flask(__name__)
-
-def call_chatgpt(prompt_file, input_text):
-    # Your actual function to call the ChatGPT API will go here
-    # For now, return a placeholder string
-    with open(prompt_file, 'r') as file:
-        prompt = file.read()
-    return f"Prompt: {prompt}\nInput: {input_text}"
-
-def call_convert_script(input_text):
-    # Your actual function to call the script will go here
-    # For now, return a placeholder string
-        # Run the script and capture the output
-    process = subprocess.Popen(['python', '../convert_haml_to_py_code.py'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
-    output, error = process.communicate(input=input_text)
-    return output + error
-
-@app.route('/', methods=['GET', 'POST'])
-def index():
-    if request.method == 'POST':
-        input_text = request.form.get('InputTextbox', '')
-        script_text = request.form.get('ScriptTextbox', '')
-        haml_text = request.form.get('HAMLTextbox', '')
-        python_code_text = request.form.get('PythonCodeTextbox', '')
-        if 'TextToScriptButton' in request.form:
-            script_text = call_chatgpt('../prompts/text_to_audio_script.prompt', input_text)
-
-        elif 'ScriptToHAMLButton' in request.form:
-            haml_text = call_chatgpt('../prompts/audio_script_to_HAML.prompt', script_text)
-
-        elif 'HAMLToPythonCodeButton' in request.form:
-            python_code_text = call_convert_script(haml_text)
-
-        return render_template('index.html', haml_text=haml_text, python_code_text=python_code_text, script_text=script_text, input_text=input_text)
-
-    return render_template('index.html')
-
-if __name__ == '__main__':
-    app.run(debug=True)

webapp/templates/index.html

@@ -1,30 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <link href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.0/css/bootstrap.min.css" rel="stylesheet">
-  </head>
-  <body>
-    <div class="container">
-      <form method="POST">
-        <div class="form-group">
-          <textarea type="text" class="form-control" name="InputTextbox" placeholder="Enter text" style="word-wrap: break-word;"> {{ input_text }}</textarea>
-          <button type="submit" class="btn btn-primary" name="TextToScriptButton">Text -> Script</button>
-        </div>
-        <div class="form-group">
-          <textarea class="form-control" rows="5" name="ScriptTextbox" placeholder="Script Output" style="word-wrap: break-word;">{{ script_text }}</textarea>
-          <button type="submit" class="btn btn-primary" name="ScriptToHAMLButton">Script -> HAML</button>
-        </div>
-        <hr>
-        <div class="form-group">
-          <textarea class="form-control" rows="5" name="HAMLTextbox" placeholder="HAML Output" style="word-wrap: break-word;">{{ haml_text }}</textarea>
-          <button type="submit" class="btn btn-primary" name="HAMLToPythonCodeButton">HAML -> Python Code</button>
-        </div>
-        <hr>
-        <div class="form-group">
-          <textarea class="form-control" rows="5" name="PythonCodeTextbox" placeholder="Python Code Output" style="word-wrap: break-word;">{{ python_code_text }}</textarea>
-        </div>
-      </form>
-    </div>
-    <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.5.0/js/bootstrap.min.js"></script>
-  </body>
-</html>