File size: 10,691 Bytes
4ac3fe7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
16ddf32
4ac3fe7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6ad1e71
b67aeec
4ac3fe7
 
 
 
 
 
 
6f85aee
4ac3fe7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b67aeec
4ac3fe7
 
 
 
 
 
 
 
 
08fda23
4ac3fe7
 
 
 
 
 
 
b67aeec
bff1af0
b67aeec
 
4ac3fe7
4b2dd38
4ac3fe7
 
 
 
 
4b2dd38
4ac3fe7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
import os

os.system("pip install git+https://github.com/suno-ai/bark.git")

from bark.generation import SUPPORTED_LANGS
from bark import SAMPLE_RATE, generate_audio
from scipy.io.wavfile import write as write_wav
from datetime import datetime

import shutil
import gradio as gr

import sys

import string
import time
import argparse
import json

import numpy as np
# import IPython
# from IPython.display import Audio

import torch

from TTS.tts.utils.synthesis import synthesis
from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols
try:
  from TTS.utils.audio import AudioProcessor
except:
  from TTS.utils.audio import AudioProcessor


from TTS.tts.models import setup_model
from TTS.config import load_config
from TTS.tts.models.vits import *

from TTS.tts.utils.speakers import SpeakerManager
from pydub import AudioSegment

# from google.colab import files
import librosa

from scipy.io.wavfile import write, read

import subprocess

'''
from google.colab import drive
drive.mount('/content/drive')
src_path = os.path.join(os.path.join(os.path.join(os.path.join(os.getcwd(), 'drive'), 'MyDrive'), 'Colab Notebooks'), 'best_model_latest.pth.tar')
dst_path = os.path.join(os.getcwd(), 'best_model.pth.tar')
shutil.copy(src_path, dst_path)
'''

TTS_PATH = "TTS/"

# add libraries into environment
sys.path.append(TTS_PATH) # set this if TTS is not installed globally

# Paths definition

OUT_PATH = 'out/'

# create output path
os.makedirs(OUT_PATH, exist_ok=True)

# model vars 
MODEL_PATH = 'best_model.pth.tar'
CONFIG_PATH = 'config.json'
TTS_LANGUAGES = "language_ids.json"
TTS_SPEAKERS = "speakers.json"
USE_CUDA = torch.cuda.is_available()

# load the config
C = load_config(CONFIG_PATH)

# load the audio processor
ap = AudioProcessor(**C.audio)

speaker_embedding = None

C.model_args['d_vector_file'] = TTS_SPEAKERS
C.model_args['use_speaker_encoder_as_loss'] = False

model = setup_model(C)
model.language_manager.set_language_ids_from_file(TTS_LANGUAGES)
# print(model.language_manager.num_languages, model.embedded_language_dim)
# print(model.emb_l)
cp = torch.load(MODEL_PATH, map_location=torch.device('cpu'))
# remove speaker encoder
model_weights = cp['model'].copy()
for key in list(model_weights.keys()):
  if "speaker_encoder" in key:
    del model_weights[key]

model.load_state_dict(model_weights)

model.eval()

if USE_CUDA:
    model = model.cuda()

# synthesize voice
use_griffin_lim = False

# Paths definition

CONFIG_SE_PATH = "config_se.json"
CHECKPOINT_SE_PATH = "SE_checkpoint.pth.tar"

# Load the Speaker encoder

SE_speaker_manager = SpeakerManager(encoder_model_path=CHECKPOINT_SE_PATH, encoder_config_path=CONFIG_SE_PATH, use_cuda=USE_CUDA)

# Define helper function

def compute_spec(ref_file):
  y, sr = librosa.load(ref_file, sr=ap.sample_rate)
  spec = ap.spectrogram(y)
  spec = torch.FloatTensor(spec).unsqueeze(0)
  return spec


def voice_conversion(ta, ra, da):

  target_audio = 'target.wav'
  reference_audio = 'reference.wav'
  driving_audio = 'driving.wav'

  write(target_audio, ta[0], ta[1])
  write(reference_audio, ra[0], ra[1])
  write(driving_audio, da[0], da[1])          

  # !ffmpeg-normalize $target_audio -nt rms -t=-27 -o $target_audio -ar 16000 -f
  # !ffmpeg-normalize $reference_audio -nt rms -t=-27 -o $reference_audio -ar 16000 -f
  # !ffmpeg-normalize $driving_audio -nt rms -t=-27 -o $driving_audio -ar 16000 -f

  files = [target_audio, reference_audio, driving_audio]

  for file in files:
      subprocess.run(["ffmpeg-normalize", file, "-nt", "rms", "-t=-27", "-o", file, "-ar", "16000", "-f"])

  # ta_ = read(target_audio)

  target_emb = SE_speaker_manager.compute_d_vector_from_clip([target_audio])
  target_emb = torch.FloatTensor(target_emb).unsqueeze(0)

  driving_emb = SE_speaker_manager.compute_d_vector_from_clip([reference_audio])
  driving_emb = torch.FloatTensor(driving_emb).unsqueeze(0)

  # Convert the voice

  driving_spec = compute_spec(driving_audio)
  y_lengths = torch.tensor([driving_spec.size(-1)])
  if USE_CUDA:
      ref_wav_voc, _, _ = model.voice_conversion(driving_spec.cuda(), y_lengths.cuda(), driving_emb.cuda(), target_emb.cuda())
      ref_wav_voc = ref_wav_voc.squeeze().cpu().detach().numpy()
  else:
      ref_wav_voc, _, _ = model.voice_conversion(driving_spec, y_lengths, driving_emb, target_emb)
      ref_wav_voc = ref_wav_voc.squeeze().detach().numpy()

  # print("Reference Audio after decoder:")
  # IPython.display.display(Audio(ref_wav_voc, rate=ap.sample_rate))

  return (ap.sample_rate, ref_wav_voc)

def generate_text_to_speech(text_prompt, selected_speaker, text_temp, waveform_temp):
    audio_array = generate_audio(text_prompt, selected_speaker, text_temp, waveform_temp)

    now = datetime.now()
    date_str = now.strftime("%m-%d-%Y")
    time_str = now.strftime("%H-%M-%S")

    outputs_folder = os.path.join(os.getcwd(), "outputs")
    if not os.path.exists(outputs_folder):
        os.makedirs(outputs_folder)

    sub_folder = os.path.join(outputs_folder, date_str)
    if not os.path.exists(sub_folder):
        os.makedirs(sub_folder)

    file_name = f"audio_{time_str}.wav"
    file_path = os.path.join(sub_folder, file_name)
    write_wav(file_path, SAMPLE_RATE, audio_array)

    return file_path


speakers_list = []

for lang, code in SUPPORTED_LANGS:
    for n in range(10):
        speakers_list.append(f"{code}_speaker_{n}")

examples1 = [["ref.wav", "Bark.wav", "Bark.wav"]]

with gr.Blocks() as demo:
    gr.Markdown(
            f""" # <center>🐶🎶🥳 - Bark with Voice Cloning</center>
            
            ### <center>🤗 - Powered by [Bark](https://huggingface.co/spaces/suno/bark) and [YourTTS](https://github.com/Edresson/YourTTS). Inspired by [bark-webui](https://github.com/makawy7/bark-webui).</center>
            1. You can duplicate and use it with a GPU: <a href="https://huggingface.co/spaces/{os.getenv('SPACE_ID')}?duplicate=true"><img style="display: inline; margin-top: 0em; margin-bottom: 0em" src="https://bit.ly/3gLdBN6" alt="Duplicate Space" /></a>
            2. First use Bark to generate audio from text and then use YourTTS to get new audio in a custom voice you like. Easy to use!
            3. For voice cloning, longer reference audio (~90s) will generally lead to better quality of the cloned speech. Also, please make sure the input audio generated by Bark is not too short.
        """
    )
    
    with gr.Row().style(equal_height=True):
        inp1 = gr.Textbox(label="Input Text", lines=4, placeholder="Enter text here...")

        inp3 = gr.Slider(
            0.1,
            1.0,
            value=0.7,
            label="Generation Temperature",
            info="1.0 more diverse, 0.1 more conservative",
        )

        inp4 = gr.Slider(
            0.1, 1.0, value=0.7, label="Waveform Temperature", info="1.0 more diverse, 0.1 more conservative"
        )
    with gr.Row().style(equal_height=True):

        inp2 = gr.Dropdown(speakers_list, value=speakers_list[1], label="Acoustic Prompt")

        button = gr.Button("Generate using Bark")
        
        out1 = gr.Audio(label="Generated Audio")
    
    button.click(generate_text_to_speech, [inp1, inp2, inp3, inp4], [out1])
    
   
    with gr.Row().style(equal_height=True):
        inp5 = gr.Audio(label="Upload Reference Audio for Voice Cloning Here")
        inp6 = out1
        inp7 = out1

        btn = gr.Button("Generate using YourTTS")
        out2 = gr.Audio(label="Generated Audio in a Custom Voice")

    btn.click(voice_conversion, [inp5, inp6, inp7], [out2])

    gr.Examples(examples=examples1, fn=voice_conversion, inputs=[inp5, inp6, inp7],
                outputs=[out2], cache_examples=True)
    
    gr.Markdown(
            """ ### <center>NOTE: Please do not generate any audio that is potentially harmful to any person or organization❗</center>
                        
        """
    )
    gr.Markdown(
            """ 
### <center>😄 - You may also apply [VoiceFixer](https://huggingface.co/spaces/Kevin676/VoiceFixer) to the generated audio in order to enhance the speech.</center>
## 🌎 Foreign Language
Bark supports various languages out-of-the-box and automatically determines language from input text. \
When prompted with code-switched text, Bark will even attempt to employ the native accent for the respective languages in the same voice.
Try the prompt:
```
Buenos días Miguel. Tu colega piensa que tu alemán es extremadamente malo. But I suppose your english isn't terrible.
```
## 🤭 Non-Speech Sounds
Below is a list of some known non-speech sounds, but we are finding more every day. \
Please let us know if you find patterns that work particularly well on Discord!
* [laughter]
* [laughs]
* [sighs]
* [music]
* [gasps]
* [clears throat]
* — or ... for hesitations
* ♪ for song lyrics
* capitalization for emphasis of a word
* MAN/WOMAN: for bias towards speaker
Try the prompt:
```
" [clears throat] Hello, my name is Suno. And, uh — and I like pizza. [laughs] But I also have other interests such as... ♪ singing ♪."
```
## 🎶 Music
Bark can generate all types of audio, and, in principle, doesn't see a difference between speech and music. \
Sometimes Bark chooses to generate text as music, but you can help it out by adding music notes around your lyrics.
Try the prompt:
```
♪ In the jungle, the mighty jungle, the lion barks tonight ♪
```
## 🧬 Voice Cloning
Bark has the capability to fully clone voices - including tone, pitch, emotion and prosody. \
The model also attempts to preserve music, ambient noise, etc. from input audio. \
However, to mitigate misuse of this technology, we limit the audio history prompts to a limited set of Suno-provided, fully synthetic options to choose from.
## 👥 Speaker Prompts
You can provide certain speaker prompts such as NARRATOR, MAN, WOMAN, etc. \
Please note that these are not always respected, especially if a conflicting audio history prompt is given.
Try the prompt:
```
WOMAN: I would like an oatmilk latte please.
MAN: Wow, that's expensive!
```
## Details
Bark model by [Suno](https://suno.ai/), including official [code](https://github.com/suno-ai/bark) and model weights. \
Gradio demo supported by 🤗 Hugging Face. Bark is licensed under a non-commercial license: CC-BY 4.0 NC, see details on [GitHub](https://github.com/suno-ai/bark).
                        
        """
    )
    
        
    gr.HTML('''
        <div class="footer">
                    <p>🎶🖼️🎡 - It’s the intersection of technology and liberal arts that makes our hearts sing — Steve Jobs
                    </p>
        </div>
    ''')     

demo.queue().launch(show_error=True)