Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,127 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Data/OOD_texts.txt filter=lfs diff=lfs merge=lfs -text
+Demo/Inference_LJSpeech.ipynb filter=lfs diff=lfs merge=lfs -text
+Demo/Inference_LibriTTS.ipynb filter=lfs diff=lfs merge=lfs -text
+Demo/reference_audio/James.wav filter=lfs diff=lfs merge=lfs -text
+Demo/reference_audio/James1.wav filter=lfs diff=lfs merge=lfs -text
+Demo/reference_audio/James2.wav filter=lfs diff=lfs merge=lfs -text
+Utils/JDC/bst.t7 filter=lfs diff=lfs merge=lfs -text
+Utils/PLBERT/step_1000000.t7 filter=lfs diff=lfs merge=lfs -text
+original_voice.wav filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/Pillow.libs/libfreetype-82733d78.so.6.20.1 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/Pillow.libs/libharfbuzz-e3b74c67.so.0.60821.0 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/_cffi_backend.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/_soundfile_data/libsndfile_x86_64.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/altair/vegalite/v5/schema/__pycache__/channels.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/altair/vegalite/v5/schema/__pycache__/core.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_cython.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/fontTools/cu2qu/cu2qu.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/fontTools/feaLib/lexer.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/fontTools/misc/bezierTools.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/fontTools/pens/momentsPen.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/fontTools/qu2cu/qu2cu.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/fontTools/varLib/iup.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/gradio/frpc_linux_amd64_v0.2 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/gradio/templates/frontend/assets/Index-c6e4b94b.js.map filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/kiwisolver/_cext.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/llvmlite/binding/libllvmlite.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/lxml/etree.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/lxml/objectify.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/matplotlib/_image.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/matplotlib/_path.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/matplotlib/_qhull.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/matplotlib/backends/_backend_agg.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/matplotlib/ft2font.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/monotonic_align/core.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/monotonic_align/core1alt.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/monotonic_align/core2.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/monotonic_align/core2eps.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/msgpack/_cmsgpack.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/numba/np/ufunc/tbbpool.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/numpy/core/_multiarray_umath.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/numpy/core/_simd.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/numpy.libs/libgfortran-040039e1.so.5.0.0 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/numpy.libs/libopenblas64_p-r0-0cf96a72.3.23.dev.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvfuser/_C.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/cublas/lib/libcublas.so.12 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/cublas/lib/libcublasLt.so.12 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/cuda_cupti/lib/libcheckpoint.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/cuda_cupti/lib/libcupti.so.12 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/cuda_cupti/lib/libnvperf_host.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/cuda_cupti/lib/libnvperf_target.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/cuda_nvrtc/lib/libnvrtc-builtins.so.12.1 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/cuda_nvrtc/lib/libnvrtc.so.12 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/cudnn/lib/libcudnn_adv_infer.so.8 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/cudnn/lib/libcudnn_adv_train.so.8 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/cudnn/lib/libcudnn_cnn_infer.so.8 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/cudnn/lib/libcudnn_cnn_train.so.8 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/cudnn/lib/libcudnn_ops_infer.so.8 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/cudnn/lib/libcudnn_ops_train.so.8 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/cufft/lib/libcufft.so.11 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/cufft/lib/libcufftw.so.11 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/curand/lib/libcurand.so.10 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/cusolver/lib/libcusolver.so.11 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/cusolver/lib/libcusolverMg.so.11 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/cusparse/lib/libcusparse.so.12 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/nccl/lib/libnccl.so.2 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/nvidia/nvjitlink/lib/libnvJitLink.so.12 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/pandas/_libs/algos.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/pandas/_libs/groupby.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/pandas/_libs/hashtable.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/pandas/_libs/interval.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/pandas/_libs/join.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/pandas/_libs/tslibs/offsets.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/pyarrow/_compute.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/pyarrow/_dataset.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/pyarrow/_flight.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/pyarrow/lib.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/pyarrow/libarrow.so.1400 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/pyarrow/libarrow_acero.so.1400 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/pyarrow/libarrow_dataset.so.1400 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/pyarrow/libarrow_flight.so.1400 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/pyarrow/libarrow_python.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/pyarrow/libarrow_substrait.so.1400 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/pyarrow/libparquet.so.1400 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/pydantic_core/_pydantic_core.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/pyzmq.libs/libsodium-cb25555f.so.23.3.0 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/pyzmq.libs/libzmq-f468291a.so.5.2.4 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/regex/_regex.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/rpds/rpds.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/safetensors/_safetensors_rust.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/scipy/fft/_pocketfft/pypocketfft.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/scipy/linalg/_flapack.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/scipy/misc/face.dat filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/scipy/optimize/_highs/_highs_wrapper.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/scipy/sparse/_sparsetools.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/scipy/spatial/_ckdtree.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/scipy/spatial/_qhull.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/scipy/special/_ufuncs.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/scipy/special/cython_special.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/scipy/stats/_unuran/unuran_wrapper.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/scipy.libs/libgfortran-040039e1.so.5.0.0 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/scipy.libs/libopenblasp-r0-23e5df77.3.21.dev.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/sklearn/_loss/_loss.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/soxr/cysoxr.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/speech_recognition/flac-linux-x86 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/speech_recognition/flac-linux-x86_64 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/speech_recognition/pocketsphinx-data/en-US/acoustic-model/mdef filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/speech_recognition/pocketsphinx-data/en-US/acoustic-model/sendump filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/sympy/polys/benchmarks/__pycache__/bench_solvers.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/tiktoken/_tiktoken.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/tokenizers/tokenizers.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/torch/bin/nvfuser_tests filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/torch/bin/protoc filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/torch/bin/protoc-3.13.0.0 filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/torch/lib/libc10.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/torch/lib/libnvfuser_codegen.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/torch/lib/libtorch_cpu.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/torch/lib/libtorch_cuda.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/torch/lib/libtorch_cuda_linalg.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/torch/lib/libtorch_python.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/torchaudio/lib/libctc_prefix_decoder.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/torchaudio/lib/libtorchaudio.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/triton/_C/libtriton.so filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/triton/third_party/cuda/bin/ptxas filter=lfs diff=lfs merge=lfs -text
+styleenv/lib/python3.11/site-packages/yaml/_yaml.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text

.ipynb_checkpoints/onefile-checkpoint.ipynb

@@ -0,0 +1,232 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "612692e6-fe5f-4787-86d9-c660bb9a21ec",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "ModuleNotFoundError",
+     "evalue": "No module named 'models'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[1], line 27\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mlibrosa\u001b[39;00m\n\u001b[1;32m     25\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mnltk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtokenize\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m word_tokenize\n\u001b[0;32m---> 27\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mmodels\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[1;32m     28\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mutils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[1;32m     29\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtext_utils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m TextCleaner\n",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'models'"
+     ]
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "torch.manual_seed(0)\n",
+    "torch.backends.cudnn.benchmark = False\n",
+    "torch.backends.cudnn.deterministic = True\n",
+    "\n",
+    "import random\n",
+    "random.seed(0)\n",
+    "\n",
+    "import numpy as np\n",
+    "np.random.seed(0)\n",
+    "\n",
+    "#%cd ..\n",
+    "\n",
+    "# load packages\n",
+    "import time\n",
+    "import random\n",
+    "import yaml\n",
+    "from munch import Munch\n",
+    "import numpy as np\n",
+    "import torch\n",
+    "from torch import nn\n",
+    "import torch.nn.functional as F\n",
+    "import torchaudio\n",
+    "import librosa\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "\n",
+    "from models import *\n",
+    "from utils import *\n",
+    "from text_utils import TextCleaner\n",
+    "textclenaer = TextCleaner()\n",
+    "\n",
+    "%matplotlib inline\n",
+    "\n",
+    "device = 'cuda'\n",
+    "\n",
+    "to_mel = torchaudio.transforms.MelSpectrogram(\n",
+    "    n_mels=80, n_fft=2048, win_length=1200, hop_length=300)\n",
+    "mean, std = -4, 4\n",
+    "\n",
+    "def length_to_mask(lengths):\n",
+    "    mask = torch.arange(lengths.max()).unsqueeze(0).expand(lengths.shape[0], -1).type_as(lengths)\n",
+    "    mask = torch.gt(mask+1, lengths.unsqueeze(1))\n",
+    "    return mask\n",
+    "\n",
+    "def preprocess(wave):\n",
+    "    wave_tensor = torch.from_numpy(wave).float()\n",
+    "    mel_tensor = to_mel(wave_tensor)\n",
+    "    mel_tensor = (torch.log(1e-5 + mel_tensor.unsqueeze(0)) - mean) / std\n",
+    "    return mel_tensor\n",
+    "\n",
+    "def compute_style(ref_dicts):\n",
+    "    reference_embeddings = {}\n",
+    "    for key, path in ref_dicts.items():\n",
+    "        wave, sr = librosa.load(path, sr=24000)\n",
+    "        audio, index = librosa.effects.trim(wave, top_db=30)\n",
+    "        if sr != 24000:\n",
+    "            audio = librosa.resample(audio, sr, 24000)\n",
+    "        mel_tensor = preprocess(audio).to(device)\n",
+    "\n",
+    "        with torch.no_grad():\n",
+    "            ref = model.style_encoder(mel_tensor.unsqueeze(1))\n",
+    "        reference_embeddings[key] = (ref.squeeze(1), audio)\n",
+    "    \n",
+    "    return reference_embeddings\n",
+    "\n",
+    "# load phonemizer\n",
+    "import phonemizer\n",
+    "global_phonemizer = phonemizer.backend.EspeakBackend(language='en-us', preserve_punctuation=True,  with_stress=True)\n",
+    "\n",
+    "config = yaml.safe_load(open(\"Models/LJSpeech/config.yml\"))\n",
+    "\n",
+    "# load pretrained ASR model\n",
+    "ASR_config = config.get('ASR_config', False)\n",
+    "ASR_path = config.get('ASR_path', False)\n",
+    "text_aligner = load_ASR_models(ASR_path, ASR_config)\n",
+    "\n",
+    "# load pretrained F0 model\n",
+    "F0_path = config.get('F0_path', False)\n",
+    "pitch_extractor = load_F0_models(F0_path)\n",
+    "\n",
+    "# load BERT model\n",
+    "from Utils.PLBERT.util import load_plbert\n",
+    "BERT_path = config.get('PLBERT_dir', False)\n",
+    "plbert = load_plbert(BERT_path)\n",
+    "\n",
+    "model = build_model(recursive_munch(config['model_params']), text_aligner, pitch_extractor, plbert)\n",
+    "_ = [model[key].eval() for key in model]\n",
+    "_ = [model[key].to(device) for key in model]\n",
+    "\n",
+    "params_whole = torch.load(\"Models/LJSpeech/epoch_2nd_00100.pth\", map_location='cpu')\n",
+    "params = params_whole['net']\n",
+    "\n",
+    "for key in model:\n",
+    "    if key in params:\n",
+    "        print('%s loaded' % key)\n",
+    "        try:\n",
+    "            model[key].load_state_dict(params[key])\n",
+    "        except:\n",
+    "            from collections import OrderedDict\n",
+    "            state_dict = params[key]\n",
+    "            new_state_dict = OrderedDict()\n",
+    "            for k, v in state_dict.items():\n",
+    "                name = k[7:] # remove `module.`\n",
+    "                new_state_dict[name] = v\n",
+    "            # load params\n",
+    "            model[key].load_state_dict(new_state_dict, strict=False)\n",
+    "#             except:\n",
+    "#                 _load(params[key], model[key])\n",
+    "_ = [model[key].eval() for key in model]\n",
+    "\n",
+    "from Modules.diffusion.sampler import DiffusionSampler, ADPM2Sampler, KarrasSchedule\n",
+    "\n",
+    "sampler = DiffusionSampler(\n",
+    "    model.diffusion.diffusion,\n",
+    "    sampler=ADPM2Sampler(),\n",
+    "    sigma_schedule=KarrasSchedule(sigma_min=0.0001, sigma_max=3.0, rho=9.0), # empirical parameters\n",
+    "    clamp=False\n",
+    ")\n",
+    "\n",
+    "def inference(text, noise, diffusion_steps=5, embedding_scale=1):\n",
+    "    text = text.strip()\n",
+    "    text = text.replace('\"', '')\n",
+    "    ps = global_phonemizer.phonemize([text])\n",
+    "    ps = word_tokenize(ps[0])\n",
+    "    ps = ' '.join(ps)\n",
+    "\n",
+    "    tokens = textclenaer(ps)\n",
+    "    tokens.insert(0, 0)\n",
+    "    tokens = torch.LongTensor(tokens).to(device).unsqueeze(0)\n",
+    "    \n",
+    "    with torch.no_grad():\n",
+    "        input_lengths = torch.LongTensor([tokens.shape[-1]]).to(tokens.device)\n",
+    "        text_mask = length_to_mask(input_lengths).to(tokens.device)\n",
+    "\n",
+    "        t_en = model.text_encoder(tokens, input_lengths, text_mask)\n",
+    "        bert_dur = model.bert(tokens, attention_mask=(~text_mask).int())\n",
+    "        d_en = model.bert_encoder(bert_dur).transpose(-1, -2) \n",
+    "\n",
+    "        s_pred = sampler(noise, \n",
+    "              embedding=bert_dur[0].unsqueeze(0), num_steps=diffusion_steps,\n",
+    "              embedding_scale=embedding_scale).squeeze(0)\n",
+    "\n",
+    "        s = s_pred[:, 128:]\n",
+    "        ref = s_pred[:, :128]\n",
+    "\n",
+    "        d = model.predictor.text_encoder(d_en, s, input_lengths, text_mask)\n",
+    "\n",
+    "        x, _ = model.predictor.lstm(d)\n",
+    "        duration = model.predictor.duration_proj(x)\n",
+    "        duration = torch.sigmoid(duration).sum(axis=-1)\n",
+    "        pred_dur = torch.round(duration.squeeze()).clamp(min=1)\n",
+    "\n",
+    "        pred_dur[-1] += 5\n",
+    "\n",
+    "        pred_aln_trg = torch.zeros(input_lengths, int(pred_dur.sum().data))\n",
+    "        c_frame = 0\n",
+    "        for i in range(pred_aln_trg.size(0)):\n",
+    "            pred_aln_trg[i, c_frame:c_frame + int(pred_dur[i].data)] = 1\n",
+    "            c_frame += int(pred_dur[i].data)\n",
+    "\n",
+    "        # encode prosody\n",
+    "        en = (d.transpose(-1, -2) @ pred_aln_trg.unsqueeze(0).to(device))\n",
+    "        F0_pred, N_pred = model.predictor.F0Ntrain(en, s)\n",
+    "        out = model.decoder((t_en @ pred_aln_trg.unsqueeze(0).to(device)), \n",
+    "                                F0_pred, N_pred, ref.squeeze().unsqueeze(0))\n",
+    "        \n",
+    "    return out.squeeze().cpu().numpy()\n",
+    "\n",
+    "# synthesize a text\n",
+    "text = ''' Hi James! How are you? '''\n",
+    "\n",
+    "start = time.time()\n",
+    "noise = torch.randn(1,1,256).to(device)\n",
+    "wav = inference(text, noise, diffusion_steps=5, embedding_scale=1)\n",
+    "rtf = (time.time() - start) / (len(wav) / 24000)\n",
+    "print(f\"RTF = {rtf:5f}\")\n",
+    "import IPython.display as ipd\n",
+    "display(ipd.Audio(wav, rate=24000))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e28c4e1a-ddb6-4914-8b26-a6053e90c272",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python (styleenv)",
+   "language": "python",
+   "name": "styleenv"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

.ipynb_checkpoints/voice_clone-checkpoint.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "git.ignoreLimitWarning": true
+}

Colab/StyleTTS2_Demo_LJSpeech.ipynb

@@ -0,0 +1,486 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "gpuType": "T4",
+      "authorship_tag": "ABX9TyM1x2mx2VnkYNFVlD+DFzmy",
+      "include_colab_link": true
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "accelerator": "GPU"
+  },
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "view-in-github",
+        "colab_type": "text"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/yl4579/StyleTTS2/blob/main/Colab/StyleTTS2_Demo_LJSpeech.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Install packages and download models"
+      ],
+      "metadata": {
+        "id": "nm653VK4CG9F"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "%%shell\n",
+        "git clone https://github.com/yl4579/StyleTTS2.git\n",
+        "cd StyleTTS2\n",
+        "pip install SoundFile torchaudio munch torch pydub pyyaml librosa nltk matplotlib accelerate transformers phonemizer einops einops-exts tqdm typing-extensions git+https://github.com/resemble-ai/monotonic_align.git\n",
+        "sudo apt-get install espeak-ng\n",
+        "git-lfs clone https://huggingface.co/yl4579/StyleTTS2-LJSpeech\n",
+        "mv StyleTTS2-LJSpeech/Models ."
+      ],
+      "metadata": {
+        "id": "gciBKMqCCLvT"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Load models"
+      ],
+      "metadata": {
+        "id": "OAA8lx-XCQnM"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "%cd StyleTTS2\n",
+        "\n",
+        "import torch\n",
+        "torch.manual_seed(0)\n",
+        "torch.backends.cudnn.benchmark = False\n",
+        "torch.backends.cudnn.deterministic = True\n",
+        "\n",
+        "import random\n",
+        "random.seed(0)\n",
+        "\n",
+        "import numpy as np\n",
+        "np.random.seed(0)\n",
+        "\n",
+        "import nltk\n",
+        "nltk.download('punkt')\n",
+        "\n",
+        "# load packages\n",
+        "import time\n",
+        "import random\n",
+        "import yaml\n",
+        "from munch import Munch\n",
+        "import numpy as np\n",
+        "import torch\n",
+        "from torch import nn\n",
+        "import torch.nn.functional as F\n",
+        "import torchaudio\n",
+        "import librosa\n",
+        "from nltk.tokenize import word_tokenize\n",
+        "\n",
+        "from models import *\n",
+        "from utils import *\n",
+        "from text_utils import TextCleaner\n",
+        "textclenaer = TextCleaner()\n",
+        "\n",
+        "%matplotlib inline\n",
+        "\n",
+        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
+        "\n",
+        "to_mel = torchaudio.transforms.MelSpectrogram(\n",
+        "    n_mels=80, n_fft=2048, win_length=1200, hop_length=300)\n",
+        "mean, std = -4, 4\n",
+        "\n",
+        "def length_to_mask(lengths):\n",
+        "    mask = torch.arange(lengths.max()).unsqueeze(0).expand(lengths.shape[0], -1).type_as(lengths)\n",
+        "    mask = torch.gt(mask+1, lengths.unsqueeze(1))\n",
+        "    return mask\n",
+        "\n",
+        "def preprocess(wave):\n",
+        "    wave_tensor = torch.from_numpy(wave).float()\n",
+        "    mel_tensor = to_mel(wave_tensor)\n",
+        "    mel_tensor = (torch.log(1e-5 + mel_tensor.unsqueeze(0)) - mean) / std\n",
+        "    return mel_tensor\n",
+        "\n",
+        "def compute_style(ref_dicts):\n",
+        "    reference_embeddings = {}\n",
+        "    for key, path in ref_dicts.items():\n",
+        "        wave, sr = librosa.load(path, sr=24000)\n",
+        "        audio, index = librosa.effects.trim(wave, top_db=30)\n",
+        "        if sr != 24000:\n",
+        "            audio = librosa.resample(audio, sr, 24000)\n",
+        "        mel_tensor = preprocess(audio).to(device)\n",
+        "\n",
+        "        with torch.no_grad():\n",
+        "            ref = model.style_encoder(mel_tensor.unsqueeze(1))\n",
+        "        reference_embeddings[key] = (ref.squeeze(1), audio)\n",
+        "\n",
+        "    return reference_embeddings\n",
+        "\n",
+        "# load phonemizer\n",
+        "import phonemizer\n",
+        "global_phonemizer = phonemizer.backend.EspeakBackend(language='en-us', preserve_punctuation=True, with_stress=True, words_mismatch='ignore')\n",
+        "\n",
+        "config = yaml.safe_load(open(\"Models/LJSpeech/config.yml\"))\n",
+        "\n",
+        "# load pretrained ASR model\n",
+        "ASR_config = config.get('ASR_config', False)\n",
+        "ASR_path = config.get('ASR_path', False)\n",
+        "text_aligner = load_ASR_models(ASR_path, ASR_config)\n",
+        "\n",
+        "# load pretrained F0 model\n",
+        "F0_path = config.get('F0_path', False)\n",
+        "pitch_extractor = load_F0_models(F0_path)\n",
+        "\n",
+        "# load BERT model\n",
+        "from Utils.PLBERT.util import load_plbert\n",
+        "BERT_path = config.get('PLBERT_dir', False)\n",
+        "plbert = load_plbert(BERT_path)\n",
+        "\n",
+        "model = build_model(recursive_munch(config['model_params']), text_aligner, pitch_extractor, plbert)\n",
+        "_ = [model[key].eval() for key in model]\n",
+        "_ = [model[key].to(device) for key in model]\n",
+        "\n",
+        "params_whole = torch.load(\"Models/LJSpeech/epoch_2nd_00100.pth\", map_location='cpu')\n",
+        "params = params_whole['net']\n",
+        "\n",
+        "for key in model:\n",
+        "    if key in params:\n",
+        "        print('%s loaded' % key)\n",
+        "        try:\n",
+        "            model[key].load_state_dict(params[key])\n",
+        "        except:\n",
+        "            from collections import OrderedDict\n",
+        "            state_dict = params[key]\n",
+        "            new_state_dict = OrderedDict()\n",
+        "            for k, v in state_dict.items():\n",
+        "                name = k[7:] # remove `module.`\n",
+        "                new_state_dict[name] = v\n",
+        "            # load params\n",
+        "            model[key].load_state_dict(new_state_dict, strict=False)\n",
+        "#             except:\n",
+        "#                 _load(params[key], model[key])\n",
+        "_ = [model[key].eval() for key in model]\n",
+        "\n",
+        "from Modules.diffusion.sampler import DiffusionSampler, ADPM2Sampler, KarrasSchedule\n",
+        "\n",
+        "sampler = DiffusionSampler(\n",
+        "    model.diffusion.diffusion,\n",
+        "    sampler=ADPM2Sampler(),\n",
+        "    sigma_schedule=KarrasSchedule(sigma_min=0.0001, sigma_max=3.0, rho=9.0), # empirical parameters\n",
+        "    clamp=False\n",
+        ")\n",
+        "\n",
+        "def inference(text, noise, diffusion_steps=5, embedding_scale=1):\n",
+        "    text = text.strip()\n",
+        "    text = text.replace('\"', '')\n",
+        "    ps = global_phonemizer.phonemize([text])\n",
+        "    ps = word_tokenize(ps[0])\n",
+        "    ps = ' '.join(ps)\n",
+        "\n",
+        "    tokens = textclenaer(ps)\n",
+        "    tokens.insert(0, 0)\n",
+        "    tokens = torch.LongTensor(tokens).to(device).unsqueeze(0)\n",
+        "\n",
+        "    with torch.no_grad():\n",
+        "        input_lengths = torch.LongTensor([tokens.shape[-1]]).to(tokens.device)\n",
+        "        text_mask = length_to_mask(input_lengths).to(tokens.device)\n",
+        "\n",
+        "        t_en = model.text_encoder(tokens, input_lengths, text_mask)\n",
+        "        bert_dur = model.bert(tokens, attention_mask=(~text_mask).int())\n",
+        "        d_en = model.bert_encoder(bert_dur).transpose(-1, -2)\n",
+        "\n",
+        "        s_pred = sampler(noise,\n",
+        "              embedding=bert_dur[0].unsqueeze(0), num_steps=diffusion_steps,\n",
+        "              embedding_scale=embedding_scale).squeeze(0)\n",
+        "\n",
+        "        s = s_pred[:, 128:]\n",
+        "        ref = s_pred[:, :128]\n",
+        "\n",
+        "        d = model.predictor.text_encoder(d_en, s, input_lengths, text_mask)\n",
+        "\n",
+        "        x, _ = model.predictor.lstm(d)\n",
+        "        duration = model.predictor.duration_proj(x)\n",
+        "        duration = torch.sigmoid(duration).sum(axis=-1)\n",
+        "        pred_dur = torch.round(duration.squeeze()).clamp(min=1)\n",
+        "\n",
+        "        pred_dur[-1] += 5\n",
+        "\n",
+        "        pred_aln_trg = torch.zeros(input_lengths, int(pred_dur.sum().data))\n",
+        "        c_frame = 0\n",
+        "        for i in range(pred_aln_trg.size(0)):\n",
+        "            pred_aln_trg[i, c_frame:c_frame + int(pred_dur[i].data)] = 1\n",
+        "            c_frame += int(pred_dur[i].data)\n",
+        "\n",
+        "        # encode prosody\n",
+        "        en = (d.transpose(-1, -2) @ pred_aln_trg.unsqueeze(0).to(device))\n",
+        "        F0_pred, N_pred = model.predictor.F0Ntrain(en, s)\n",
+        "        out = model.decoder((t_en @ pred_aln_trg.unsqueeze(0).to(device)),\n",
+        "                                F0_pred, N_pred, ref.squeeze().unsqueeze(0))\n",
+        "\n",
+        "    return out.squeeze().cpu().numpy()\n",
+        "\n",
+        "def LFinference(text, s_prev, noise, alpha=0.7, diffusion_steps=5, embedding_scale=1):\n",
+        "  text = text.strip()\n",
+        "  text = text.replace('\"', '')\n",
+        "  ps = global_phonemizer.phonemize([text])\n",
+        "  ps = word_tokenize(ps[0])\n",
+        "  ps = ' '.join(ps)\n",
+        "\n",
+        "  tokens = textclenaer(ps)\n",
+        "  tokens.insert(0, 0)\n",
+        "  tokens = torch.LongTensor(tokens).to(device).unsqueeze(0)\n",
+        "\n",
+        "  with torch.no_grad():\n",
+        "      input_lengths = torch.LongTensor([tokens.shape[-1]]).to(tokens.device)\n",
+        "      text_mask = length_to_mask(input_lengths).to(tokens.device)\n",
+        "\n",
+        "      t_en = model.text_encoder(tokens, input_lengths, text_mask)\n",
+        "      bert_dur = model.bert(tokens, attention_mask=(~text_mask).int())\n",
+        "      d_en = model.bert_encoder(bert_dur).transpose(-1, -2)\n",
+        "\n",
+        "      s_pred = sampler(noise,\n",
+        "            embedding=bert_dur[0].unsqueeze(0), num_steps=diffusion_steps,\n",
+        "            embedding_scale=embedding_scale).squeeze(0)\n",
+        "\n",
+        "      if s_prev is not None:\n",
+        "          # convex combination of previous and current style\n",
+        "          s_pred = alpha * s_prev + (1 - alpha) * s_pred\n",
+        "\n",
+        "      s = s_pred[:, 128:]\n",
+        "      ref = s_pred[:, :128]\n",
+        "\n",
+        "      d = model.predictor.text_encoder(d_en, s, input_lengths, text_mask)\n",
+        "\n",
+        "      x, _ = model.predictor.lstm(d)\n",
+        "      duration = model.predictor.duration_proj(x)\n",
+        "      duration = torch.sigmoid(duration).sum(axis=-1)\n",
+        "      pred_dur = torch.round(duration.squeeze()).clamp(min=1)\n",
+        "\n",
+        "      pred_aln_trg = torch.zeros(input_lengths, int(pred_dur.sum().data))\n",
+        "      c_frame = 0\n",
+        "      for i in range(pred_aln_trg.size(0)):\n",
+        "          pred_aln_trg[i, c_frame:c_frame + int(pred_dur[i].data)] = 1\n",
+        "          c_frame += int(pred_dur[i].data)\n",
+        "\n",
+        "      # encode prosody\n",
+        "      en = (d.transpose(-1, -2) @ pred_aln_trg.unsqueeze(0).to(device))\n",
+        "      F0_pred, N_pred = model.predictor.F0Ntrain(en, s)\n",
+        "      out = model.decoder((t_en @ pred_aln_trg.unsqueeze(0).to(device)),\n",
+        "                              F0_pred, N_pred, ref.squeeze().unsqueeze(0))\n",
+        "\n",
+        "  return out.squeeze().cpu().numpy(), s_pred"
+      ],
+      "metadata": {
+        "id": "m0XRpbxSCSix"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Synthesize speech"
+      ],
+      "metadata": {
+        "id": "vuCbS0gdArgJ"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# @title Input Text { display-mode: \"form\" }\n",
+        "# synthesize a text\n",
+        "text = \"StyleTTS 2 is a text-to-speech model that leverages style diffusion and adversarial training with large speech language models to achieve human-level text-to-speech synthesis.\" # @param {type:\"string\"}\n"
+      ],
+      "metadata": {
+        "id": "7Ud1Y-kbBPTw"
+      },
+      "execution_count": 3,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "#### Basic synthesis (5 diffusion steps)"
+      ],
+      "metadata": {
+        "id": "TM2NjuM7B6sz"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "start = time.time()\n",
+        "noise = torch.randn(1,1,256).to(device)\n",
+        "wav = inference(text, noise, diffusion_steps=5, embedding_scale=1)\n",
+        "rtf = (time.time() - start) / (len(wav) / 24000)\n",
+        "print(f\"RTF = {rtf:5f}\")\n",
+        "import IPython.display as ipd\n",
+        "display(ipd.Audio(wav, rate=24000))"
+      ],
+      "metadata": {
+        "id": "KILqC-V-Ay5e"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "#### With higher diffusion steps (more diverse)\n",
+        "Since the sampler is ancestral, the higher the stpes, the more diverse the samples are, with the cost of slower synthesis speed."
+      ],
+      "metadata": {
+        "id": "oZk9o-EzCBVx"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "start = time.time()\n",
+        "noise = torch.randn(1,1,256).to(device)\n",
+        "wav = inference(text, noise, diffusion_steps=10, embedding_scale=1)\n",
+        "rtf = (time.time() - start) / (len(wav) / 24000)\n",
+        "print(f\"RTF = {rtf:5f}\")\n",
+        "import IPython.display as ipd\n",
+        "display(ipd.Audio(wav, rate=24000))"
+      ],
+      "metadata": {
+        "id": "9_OHtzMbB9gL"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Speech expressiveness\n",
+        "The following section recreates the samples shown in [Section 6](https://styletts2.github.io/#emo) of the demo page."
+      ],
+      "metadata": {
+        "id": "NyDACd-0CaqL"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "#### With embedding_scale=1\n",
+        "This is the classifier-free guidance scale. The higher the scale, the more conditional the style is to the input text and hence more emotional."
+      ],
+      "metadata": {
+        "id": "cRkS5VWxCck4"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "texts = {}\n",
+        "texts['Happy'] = \"We are happy to invite you to join us on a journey to the past, where we will visit the most amazing monuments ever built by human hands.\"\n",
+        "texts['Sad'] = \"I am sorry to say that we have suffered a severe setback in our efforts to restore prosperity and confidence.\"\n",
+        "texts['Angry'] = \"The field of astronomy is a joke! Its theories are based on flawed observations and biased interpretations!\"\n",
+        "texts['Surprised'] = \"I can't believe it! You mean to tell me that you have discovered a new species of bacteria in this pond?\"\n",
+        "\n",
+        "for k,v in texts.items():\n",
+        "    noise = torch.randn(1,1,256).to(device)\n",
+        "    wav = inference(v, noise, diffusion_steps=10, embedding_scale=1)\n",
+        "    print(k + \": \")\n",
+        "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+      ],
+      "metadata": {
+        "id": "H5g5RO-mCbZB"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "#### With embedding_scale=2"
+      ],
+      "metadata": {
+        "id": "f4S8TXSpCgpA"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "texts = {}\n",
+        "texts['Happy'] = \"We are happy to invite you to join us on a journey to the past, where we will visit the most amazing monuments ever built by human hands.\"\n",
+        "texts['Sad'] = \"I am sorry to say that we have suffered a severe setback in our efforts to restore prosperity and confidence.\"\n",
+        "texts['Angry'] = \"The field of astronomy is a joke! Its theories are based on flawed observations and biased interpretations!\"\n",
+        "texts['Surprised'] = \"I can't believe it! You mean to tell me that you have discovered a new species of bacteria in this pond?\"\n",
+        "\n",
+        "for k,v in texts.items():\n",
+        "    noise = torch.randn(1,1,256).to(device)\n",
+        "    wav = inference(v, noise, diffusion_steps=10, embedding_scale=2) # embedding_scale=2 for more pronounced emotion\n",
+        "    print(k + \": \")\n",
+        "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+      ],
+      "metadata": {
+        "id": "xHHIdeNrCezC"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Long-form generation\n",
+        "This section includes basic implementation of Algorithm 1 in the paper for consistent longform audio generation. The example passage is taken from [Section 5](https://styletts2.github.io/#long) of the demo page."
+      ],
+      "metadata": {
+        "id": "nAh7Tov4CkuH"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "passage = '''If the supply of fruit is greater than the family needs, it may be made a source of income by sending the fresh fruit to the market if there is one near enough, or by preserving, canning, and making jelly for sale. To make such an enterprise a success the fruit and work must be first class. There is magic in the word \"Homemade,\" when the product appeals to the eye and the palate; but many careless and incompetent people have found to their sorrow that this word has not magic enough to float inferior goods on the market. As a rule large canning and preserving establishments are clean and have the best appliances, and they employ chemists and skilled labor. The home product must be very good to compete with the attractive goods that are sent out from such establishments. Yet for first-class homemade products there is a market in all large cities. All first-class grocers have customers who purchase such goods.''' # @param {type:\"string\"}"
+      ],
+      "metadata": {
+        "cellView": "form",
+        "id": "IJwUbgvACoDu"
+      },
+      "execution_count": 8,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "sentences = passage.split('.') # simple split by comma\n",
+        "wavs = []\n",
+        "s_prev = None\n",
+        "for text in sentences:\n",
+        "    if text.strip() == \"\": continue\n",
+        "    text += '.' # add it back\n",
+        "    noise = torch.randn(1,1,256).to(device)\n",
+        "    wav, s_prev = LFinference(text, s_prev, noise, alpha=0.7, diffusion_steps=10, embedding_scale=1.5)\n",
+        "    wavs.append(wav)\n",
+        "display(ipd.Audio(np.concatenate(wavs), rate=24000, normalize=False))"
+      ],
+      "metadata": {
+        "id": "nP-7i2QAC0JT"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ]
+}

Colab/StyleTTS2_Demo_LibriTTS.ipynb

@@ -0,0 +1,1218 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "view-in-github",
+        "colab_type": "text"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/yl4579/StyleTTS2/blob/main/Colab/StyleTTS2_Demo_LibriTTS.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aAGQPfgYIR23"
+      },
+      "source": [
+        "### Install packages and download models"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "zDPW5uSpISd2",
+        "outputId": "6463ff79-18d5-4071-c6ad-01947beeb368"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+
+          ]
+        }
+      ],
+      "source": [
+        "%%shell\n",
+        "git clone https://github.com/yl4579/StyleTTS2.git\n",
+        "cd StyleTTS2\n",
+        "pip install SoundFile torchaudio munch torch pydub pyyaml librosa nltk matplotlib accelerate transformers phonemizer einops einops-exts tqdm typing-extensions git+https://github.com/resemble-ai/monotonic_align.git\n",
+        "sudo apt-get install espeak-ng\n",
+        "git-lfs clone https://huggingface.co/yl4579/StyleTTS2-LibriTTS\n",
+        "mv StyleTTS2-LibriTTS/Models .\n",
+        "mv StyleTTS2-LibriTTS/reference_audio.zip .\n",
+        "unzip reference_audio.zip\n",
+        "mv reference_audio Demo/reference_audio"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eJdB_nCOIVIN"
+      },
+      "source": [
+        "### Load models"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cha8Tr2uJwN0"
+      },
+      "outputs": [],
+      "source": [
+        "import nltk\n",
+        "nltk.download('punkt')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Qoow8Wd8ITtm"
+      },
+      "outputs": [],
+      "source": [
+        "%cd StyleTTS2\n",
+        "\n",
+        "import torch\n",
+        "torch.manual_seed(0)\n",
+        "torch.backends.cudnn.benchmark = False\n",
+        "torch.backends.cudnn.deterministic = True\n",
+        "\n",
+        "import random\n",
+        "random.seed(0)\n",
+        "\n",
+        "import numpy as np\n",
+        "np.random.seed(0)\n",
+        "\n",
+        "# load packages\n",
+        "import time\n",
+        "import random\n",
+        "import yaml\n",
+        "from munch import Munch\n",
+        "import numpy as np\n",
+        "import torch\n",
+        "from torch import nn\n",
+        "import torch.nn.functional as F\n",
+        "import torchaudio\n",
+        "import librosa\n",
+        "from nltk.tokenize import word_tokenize\n",
+        "\n",
+        "from models import *\n",
+        "from utils import *\n",
+        "from text_utils import TextCleaner\n",
+        "textclenaer = TextCleaner()\n",
+        "\n",
+        "%matplotlib inline\n",
+        "\n",
+        "to_mel = torchaudio.transforms.MelSpectrogram(\n",
+        "    n_mels=80, n_fft=2048, win_length=1200, hop_length=300)\n",
+        "mean, std = -4, 4\n",
+        "\n",
+        "def length_to_mask(lengths):\n",
+        "    mask = torch.arange(lengths.max()).unsqueeze(0).expand(lengths.shape[0], -1).type_as(lengths)\n",
+        "    mask = torch.gt(mask+1, lengths.unsqueeze(1))\n",
+        "    return mask\n",
+        "\n",
+        "def preprocess(wave):\n",
+        "    wave_tensor = torch.from_numpy(wave).float()\n",
+        "    mel_tensor = to_mel(wave_tensor)\n",
+        "    mel_tensor = (torch.log(1e-5 + mel_tensor.unsqueeze(0)) - mean) / std\n",
+        "    return mel_tensor\n",
+        "\n",
+        "def compute_style(path):\n",
+        "    wave, sr = librosa.load(path, sr=24000)\n",
+        "    audio, index = librosa.effects.trim(wave, top_db=30)\n",
+        "    if sr != 24000:\n",
+        "        audio = librosa.resample(audio, sr, 24000)\n",
+        "    mel_tensor = preprocess(audio).to(device)\n",
+        "\n",
+        "    with torch.no_grad():\n",
+        "        ref_s = model.style_encoder(mel_tensor.unsqueeze(1))\n",
+        "        ref_p = model.predictor_encoder(mel_tensor.unsqueeze(1))\n",
+        "\n",
+        "    return torch.cat([ref_s, ref_p], dim=1)\n",
+        "\n",
+        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
+        "\n",
+        "# load phonemizer\n",
+        "import phonemizer\n",
+        "global_phonemizer = phonemizer.backend.EspeakBackend(language='en-us', preserve_punctuation=True,  with_stress=True)\n",
+        "\n",
+        "config = yaml.safe_load(open(\"Models/LibriTTS/config.yml\"))\n",
+        "\n",
+        "# load pretrained ASR model\n",
+        "ASR_config = config.get('ASR_config', False)\n",
+        "ASR_path = config.get('ASR_path', False)\n",
+        "text_aligner = load_ASR_models(ASR_path, ASR_config)\n",
+        "\n",
+        "# load pretrained F0 model\n",
+        "F0_path = config.get('F0_path', False)\n",
+        "pitch_extractor = load_F0_models(F0_path)\n",
+        "\n",
+        "# load BERT model\n",
+        "from Utils.PLBERT.util import load_plbert\n",
+        "BERT_path = config.get('PLBERT_dir', False)\n",
+        "plbert = load_plbert(BERT_path)\n",
+        "\n",
+        "model_params = recursive_munch(config['model_params'])\n",
+        "model = build_model(model_params, text_aligner, pitch_extractor, plbert)\n",
+        "_ = [model[key].eval() for key in model]\n",
+        "_ = [model[key].to(device) for key in model]\n",
+        "\n",
+        "params_whole = torch.load(\"Models/LibriTTS/epochs_2nd_00020.pth\", map_location='cpu')\n",
+        "params = params_whole['net']\n",
+        "\n",
+        "for key in model:\n",
+        "    if key in params:\n",
+        "        print('%s loaded' % key)\n",
+        "        try:\n",
+        "            model[key].load_state_dict(params[key])\n",
+        "        except:\n",
+        "            from collections import OrderedDict\n",
+        "            state_dict = params[key]\n",
+        "            new_state_dict = OrderedDict()\n",
+        "            for k, v in state_dict.items():\n",
+        "                name = k[7:] # remove `module.`\n",
+        "                new_state_dict[name] = v\n",
+        "            # load params\n",
+        "            model[key].load_state_dict(new_state_dict, strict=False)\n",
+        "#             except:\n",
+        "#                 _load(params[key], model[key])\n",
+        "_ = [model[key].eval() for key in model]\n",
+        "\n",
+        "from Modules.diffusion.sampler import DiffusionSampler, ADPM2Sampler, KarrasSchedule\n",
+        "\n",
+        "sampler = DiffusionSampler(\n",
+        "    model.diffusion.diffusion,\n",
+        "    sampler=ADPM2Sampler(),\n",
+        "    sigma_schedule=KarrasSchedule(sigma_min=0.0001, sigma_max=3.0, rho=9.0), # empirical parameters\n",
+        "    clamp=False\n",
+        ")\n",
+        "\n",
+        "def inference(text, ref_s, alpha = 0.3, beta = 0.7, diffusion_steps=5, embedding_scale=1):\n",
+        "    text = text.strip()\n",
+        "    ps = global_phonemizer.phonemize([text])\n",
+        "    ps = word_tokenize(ps[0])\n",
+        "    ps = ' '.join(ps)\n",
+        "    tokens = textclenaer(ps)\n",
+        "    tokens.insert(0, 0)\n",
+        "    tokens = torch.LongTensor(tokens).to(device).unsqueeze(0)\n",
+        "\n",
+        "    with torch.no_grad():\n",
+        "        input_lengths = torch.LongTensor([tokens.shape[-1]]).to(device)\n",
+        "        text_mask = length_to_mask(input_lengths).to(device)\n",
+        "\n",
+        "        t_en = model.text_encoder(tokens, input_lengths, text_mask)\n",
+        "        bert_dur = model.bert(tokens, attention_mask=(~text_mask).int())\n",
+        "        d_en = model.bert_encoder(bert_dur).transpose(-1, -2)\n",
+        "\n",
+        "        s_pred = sampler(noise = torch.randn((1, 256)).unsqueeze(1).to(device),\n",
+        "                                          embedding=bert_dur,\n",
+        "                                          embedding_scale=embedding_scale,\n",
+        "                                            features=ref_s, # reference from the same speaker as the embedding\n",
+        "                                             num_steps=diffusion_steps).squeeze(1)\n",
+        "\n",
+        "\n",
+        "        s = s_pred[:, 128:]\n",
+        "        ref = s_pred[:, :128]\n",
+        "\n",
+        "        ref = alpha * ref + (1 - alpha)  * ref_s[:, :128]\n",
+        "        s = beta * s + (1 - beta)  * ref_s[:, 128:]\n",
+        "\n",
+        "        d = model.predictor.text_encoder(d_en,\n",
+        "                                         s, input_lengths, text_mask)\n",
+        "\n",
+        "        x, _ = model.predictor.lstm(d)\n",
+        "        duration = model.predictor.duration_proj(x)\n",
+        "\n",
+        "        duration = torch.sigmoid(duration).sum(axis=-1)\n",
+        "        pred_dur = torch.round(duration.squeeze()).clamp(min=1)\n",
+        "\n",
+        "\n",
+        "        pred_aln_trg = torch.zeros(input_lengths, int(pred_dur.sum().data))\n",
+        "        c_frame = 0\n",
+        "        for i in range(pred_aln_trg.size(0)):\n",
+        "            pred_aln_trg[i, c_frame:c_frame + int(pred_dur[i].data)] = 1\n",
+        "            c_frame += int(pred_dur[i].data)\n",
+        "\n",
+        "        # encode prosody\n",
+        "        en = (d.transpose(-1, -2) @ pred_aln_trg.unsqueeze(0).to(device))\n",
+        "        if model_params.decoder.type == \"hifigan\":\n",
+        "            asr_new = torch.zeros_like(en)\n",
+        "            asr_new[:, :, 0] = en[:, :, 0]\n",
+        "            asr_new[:, :, 1:] = en[:, :, 0:-1]\n",
+        "            en = asr_new\n",
+        "\n",
+        "        F0_pred, N_pred = model.predictor.F0Ntrain(en, s)\n",
+        "\n",
+        "        asr = (t_en @ pred_aln_trg.unsqueeze(0).to(device))\n",
+        "        if model_params.decoder.type == \"hifigan\":\n",
+        "            asr_new = torch.zeros_like(asr)\n",
+        "            asr_new[:, :, 0] = asr[:, :, 0]\n",
+        "            asr_new[:, :, 1:] = asr[:, :, 0:-1]\n",
+        "            asr = asr_new\n",
+        "\n",
+        "        out = model.decoder(asr,\n",
+        "                                F0_pred, N_pred, ref.squeeze().unsqueeze(0))\n",
+        "\n",
+        "\n",
+        "    return out.squeeze().cpu().numpy()[..., :-50] # weird pulse at the end of the model, need to be fixed later\n",
+        "\n",
+        "def LFinference(text, s_prev, ref_s, alpha = 0.3, beta = 0.7, t = 0.7, diffusion_steps=5, embedding_scale=1):\n",
+        "  text = text.strip()\n",
+        "  ps = global_phonemizer.phonemize([text])\n",
+        "  ps = word_tokenize(ps[0])\n",
+        "  ps = ' '.join(ps)\n",
+        "  ps = ps.replace('``', '\"')\n",
+        "  ps = ps.replace(\"''\", '\"')\n",
+        "\n",
+        "  tokens = textclenaer(ps)\n",
+        "  tokens.insert(0, 0)\n",
+        "  tokens = torch.LongTensor(tokens).to(device).unsqueeze(0)\n",
+        "\n",
+        "  with torch.no_grad():\n",
+        "      input_lengths = torch.LongTensor([tokens.shape[-1]]).to(device)\n",
+        "      text_mask = length_to_mask(input_lengths).to(device)\n",
+        "\n",
+        "      t_en = model.text_encoder(tokens, input_lengths, text_mask)\n",
+        "      bert_dur = model.bert(tokens, attention_mask=(~text_mask).int())\n",
+        "      d_en = model.bert_encoder(bert_dur).transpose(-1, -2)\n",
+        "\n",
+        "      s_pred = sampler(noise = torch.randn((1, 256)).unsqueeze(1).to(device),\n",
+        "                                        embedding=bert_dur,\n",
+        "                                        embedding_scale=embedding_scale,\n",
+        "                                          features=ref_s, # reference from the same speaker as the embedding\n",
+        "                                            num_steps=diffusion_steps).squeeze(1)\n",
+        "\n",
+        "      if s_prev is not None:\n",
+        "          # convex combination of previous and current style\n",
+        "          s_pred = t * s_prev + (1 - t) * s_pred\n",
+        "\n",
+        "      s = s_pred[:, 128:]\n",
+        "      ref = s_pred[:, :128]\n",
+        "\n",
+        "      ref = alpha * ref + (1 - alpha)  * ref_s[:, :128]\n",
+        "      s = beta * s + (1 - beta)  * ref_s[:, 128:]\n",
+        "\n",
+        "      s_pred = torch.cat([ref, s], dim=-1)\n",
+        "\n",
+        "      d = model.predictor.text_encoder(d_en,\n",
+        "                                        s, input_lengths, text_mask)\n",
+        "\n",
+        "      x, _ = model.predictor.lstm(d)\n",
+        "      duration = model.predictor.duration_proj(x)\n",
+        "\n",
+        "      duration = torch.sigmoid(duration).sum(axis=-1)\n",
+        "      pred_dur = torch.round(duration.squeeze()).clamp(min=1)\n",
+        "\n",
+        "\n",
+        "      pred_aln_trg = torch.zeros(input_lengths, int(pred_dur.sum().data))\n",
+        "      c_frame = 0\n",
+        "      for i in range(pred_aln_trg.size(0)):\n",
+        "          pred_aln_trg[i, c_frame:c_frame + int(pred_dur[i].data)] = 1\n",
+        "          c_frame += int(pred_dur[i].data)\n",
+        "\n",
+        "      # encode prosody\n",
+        "      en = (d.transpose(-1, -2) @ pred_aln_trg.unsqueeze(0).to(device))\n",
+        "      if model_params.decoder.type == \"hifigan\":\n",
+        "          asr_new = torch.zeros_like(en)\n",
+        "          asr_new[:, :, 0] = en[:, :, 0]\n",
+        "          asr_new[:, :, 1:] = en[:, :, 0:-1]\n",
+        "          en = asr_new\n",
+        "\n",
+        "      F0_pred, N_pred = model.predictor.F0Ntrain(en, s)\n",
+        "\n",
+        "      asr = (t_en @ pred_aln_trg.unsqueeze(0).to(device))\n",
+        "      if model_params.decoder.type == \"hifigan\":\n",
+        "          asr_new = torch.zeros_like(asr)\n",
+        "          asr_new[:, :, 0] = asr[:, :, 0]\n",
+        "          asr_new[:, :, 1:] = asr[:, :, 0:-1]\n",
+        "          asr = asr_new\n",
+        "\n",
+        "      out = model.decoder(asr,\n",
+        "                              F0_pred, N_pred, ref.squeeze().unsqueeze(0))\n",
+        "\n",
+        "\n",
+        "  return out.squeeze().cpu().numpy()[..., :-100], s_pred # weird pulse at the end of the model, need to be fixed later\n",
+        "\n",
+        "def STinference(text, ref_s, ref_text, alpha = 0.3, beta = 0.7, diffusion_steps=5, embedding_scale=1):\n",
+        "    text = text.strip()\n",
+        "    ps = global_phonemizer.phonemize([text])\n",
+        "    ps = word_tokenize(ps[0])\n",
+        "    ps = ' '.join(ps)\n",
+        "\n",
+        "    tokens = textclenaer(ps)\n",
+        "    tokens.insert(0, 0)\n",
+        "    tokens = torch.LongTensor(tokens).to(device).unsqueeze(0)\n",
+        "\n",
+        "    ref_text = ref_text.strip()\n",
+        "    ps = global_phonemizer.phonemize([ref_text])\n",
+        "    ps = word_tokenize(ps[0])\n",
+        "    ps = ' '.join(ps)\n",
+        "\n",
+        "    ref_tokens = textclenaer(ps)\n",
+        "    ref_tokens.insert(0, 0)\n",
+        "    ref_tokens = torch.LongTensor(ref_tokens).to(device).unsqueeze(0)\n",
+        "\n",
+        "\n",
+        "    with torch.no_grad():\n",
+        "        input_lengths = torch.LongTensor([tokens.shape[-1]]).to(device)\n",
+        "        text_mask = length_to_mask(input_lengths).to(device)\n",
+        "\n",
+        "        t_en = model.text_encoder(tokens, input_lengths, text_mask)\n",
+        "        bert_dur = model.bert(tokens, attention_mask=(~text_mask).int())\n",
+        "        d_en = model.bert_encoder(bert_dur).transpose(-1, -2)\n",
+        "\n",
+        "        ref_input_lengths = torch.LongTensor([ref_tokens.shape[-1]]).to(device)\n",
+        "        ref_text_mask = length_to_mask(ref_input_lengths).to(device)\n",
+        "        ref_bert_dur = model.bert(ref_tokens, attention_mask=(~ref_text_mask).int())\n",
+        "        s_pred = sampler(noise = torch.randn((1, 256)).unsqueeze(1).to(device),\n",
+        "                                          embedding=bert_dur,\n",
+        "                                          embedding_scale=embedding_scale,\n",
+        "                                            features=ref_s, # reference from the same speaker as the embedding\n",
+        "                                             num_steps=diffusion_steps).squeeze(1)\n",
+        "\n",
+        "\n",
+        "        s = s_pred[:, 128:]\n",
+        "        ref = s_pred[:, :128]\n",
+        "\n",
+        "        ref = alpha * ref + (1 - alpha)  * ref_s[:, :128]\n",
+        "        s = beta * s + (1 - beta)  * ref_s[:, 128:]\n",
+        "\n",
+        "        d = model.predictor.text_encoder(d_en,\n",
+        "                                         s, input_lengths, text_mask)\n",
+        "\n",
+        "        x, _ = model.predictor.lstm(d)\n",
+        "        duration = model.predictor.duration_proj(x)\n",
+        "\n",
+        "        duration = torch.sigmoid(duration).sum(axis=-1)\n",
+        "        pred_dur = torch.round(duration.squeeze()).clamp(min=1)\n",
+        "\n",
+        "\n",
+        "        pred_aln_trg = torch.zeros(input_lengths, int(pred_dur.sum().data))\n",
+        "        c_frame = 0\n",
+        "        for i in range(pred_aln_trg.size(0)):\n",
+        "            pred_aln_trg[i, c_frame:c_frame + int(pred_dur[i].data)] = 1\n",
+        "            c_frame += int(pred_dur[i].data)\n",
+        "\n",
+        "        # encode prosody\n",
+        "        en = (d.transpose(-1, -2) @ pred_aln_trg.unsqueeze(0).to(device))\n",
+        "        if model_params.decoder.type == \"hifigan\":\n",
+        "            asr_new = torch.zeros_like(en)\n",
+        "            asr_new[:, :, 0] = en[:, :, 0]\n",
+        "            asr_new[:, :, 1:] = en[:, :, 0:-1]\n",
+        "            en = asr_new\n",
+        "\n",
+        "        F0_pred, N_pred = model.predictor.F0Ntrain(en, s)\n",
+        "\n",
+        "        asr = (t_en @ pred_aln_trg.unsqueeze(0).to(device))\n",
+        "        if model_params.decoder.type == \"hifigan\":\n",
+        "            asr_new = torch.zeros_like(asr)\n",
+        "            asr_new[:, :, 0] = asr[:, :, 0]\n",
+        "            asr_new[:, :, 1:] = asr[:, :, 0:-1]\n",
+        "            asr = asr_new\n",
+        "\n",
+        "        out = model.decoder(asr,\n",
+        "                                F0_pred, N_pred, ref.squeeze().unsqueeze(0))\n",
+        "\n",
+        "\n",
+        "    return out.squeeze().cpu().numpy()[..., :-50] # weird pulse at the end of the model, need to be fixed later\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "32S6U0LyJbCA"
+      },
+      "source": [
+        "### Synthesize speech"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ehK_0daMJdk_"
+      },
+      "source": [
+        "#### Basic synthesis (5 diffusion steps, seen speakers)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SJs2x41MJhM-"
+      },
+      "outputs": [],
+      "source": [
+        "text = ''' StyleTTS 2 is a text to speech model that leverages style diffusion and adversarial training with large speech language models to achieve human level text to speech synthesis. ''' # @param {type:\"string\"}\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "xuqIJe-IJb7A"
+      },
+      "outputs": [],
+      "source": [
+        "reference_dicts = {}\n",
+        "reference_dicts['696_92939'] = \"Demo/reference_audio/696_92939_000016_000006.wav\"\n",
+        "reference_dicts['1789_142896'] = \"Demo/reference_audio/1789_142896_000022_000005.wav\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "H3ra3IxJJmF0"
+      },
+      "outputs": [],
+      "source": [
+        "noise = torch.randn(1,1,256).to(device)\n",
+        "for k, path in reference_dicts.items():\n",
+        "    ref_s = compute_style(path)\n",
+        "    start = time.time()\n",
+        "    wav = inference(text, ref_s, alpha=0.3, beta=0.7, diffusion_steps=5, embedding_scale=1)\n",
+        "    rtf = (time.time() - start) / (len(wav) / 24000)\n",
+        "    print(f\"RTF = {rtf:5f}\")\n",
+        "    import IPython.display as ipd\n",
+        "    print(k + ' Synthesized:')\n",
+        "    display(ipd.Audio(wav, rate=24000, normalize=False))\n",
+        "    print('Reference:')\n",
+        "    display(ipd.Audio(path, rate=24000, normalize=False))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aB3wUz6yJ-P_"
+      },
+      "source": [
+        "#### With higher diffusion steps (more diverse)\n",
+        "\n",
+        "Since the sampler is ancestral, the higher the stpes, the more diverse the samples are, with the cost of slower synthesis speed."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lF27XUo4JrKk"
+      },
+      "outputs": [],
+      "source": [
+        "noise = torch.randn(1,1,256).to(device)\n",
+        "for k, path in reference_dicts.items():\n",
+        "    ref_s = compute_style(path)\n",
+        "    start = time.time()\n",
+        "    wav = inference(text, ref_s, alpha=0.3, beta=0.7, diffusion_steps=10, embedding_scale=1)\n",
+        "    rtf = (time.time() - start) / (len(wav) / 24000)\n",
+        "    print(f\"RTF = {rtf:5f}\")\n",
+        "    import IPython.display as ipd\n",
+        "    print(k + ' Synthesized:')\n",
+        "    display(ipd.Audio(wav, rate=24000, normalize=False))\n",
+        "    print(k + ' Reference:')\n",
+        "    display(ipd.Audio(path, rate=24000, normalize=False))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pFT_vmJcKDs1"
+      },
+      "source": [
+        "#### Basic synthesis (5 diffusion steps, unseen speakers)\n",
+        "The following samples are to reproduce samples in [Section 4](https://styletts2.github.io/#libri) of the demo page. All spsakers are unseen during training. You can compare the generated samples to popular zero-shot TTS models like Vall-E and NaturalSpeech 2."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HvNAeGPEKAWN"
+      },
+      "outputs": [],
+      "source": [
+        "reference_dicts = {}\n",
+        "# format: (path, text)\n",
+        "reference_dicts['1221-135767'] = (\"Demo/reference_audio/1221-135767-0014.wav\", \"Yea, his honourable worship is within, but he hath a godly minister or two with him, and likewise a leech.\")\n",
+        "reference_dicts['5639-40744'] = (\"Demo/reference_audio/5639-40744-0020.wav\", \"Thus did this humane and right minded father comfort his unhappy daughter, and her mother embracing her again, did all she could to soothe her feelings.\")\n",
+        "reference_dicts['908-157963'] = (\"Demo/reference_audio/908-157963-0027.wav\", \"And lay me down in my cold bed and leave my shining lot.\")\n",
+        "reference_dicts['4077-13754'] = (\"Demo/reference_audio/4077-13754-0000.wav\", \"The army found the people in poverty and left them in comparative wealth.\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mFnyvYp5KAYN"
+      },
+      "outputs": [],
+      "source": [
+        "noise = torch.randn(1,1,256).to(device)\n",
+        "for k, v in reference_dicts.items():\n",
+        "    path, text = v\n",
+        "    ref_s = compute_style(path)\n",
+        "    start = time.time()\n",
+        "    wav = inference(text, ref_s, alpha=0.3, beta=0.7, diffusion_steps=5, embedding_scale=1)\n",
+        "    rtf = (time.time() - start) / (len(wav) / 24000)\n",
+        "    print(f\"RTF = {rtf:5f}\")\n",
+        "    import IPython.display as ipd\n",
+        "    print(k + ' Synthesized: ' + text)\n",
+        "    display(ipd.Audio(wav, rate=24000, normalize=False))\n",
+        "    print(k + ' Reference:')\n",
+        "    display(ipd.Audio(path, rate=24000, normalize=False))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QBZ53BQtKNQ6"
+      },
+      "source": [
+        "### Speech expressiveness\n",
+        "\n",
+        "The following section recreates the samples shown in [Section 6](https://styletts2.github.io/#emo) of the demo page. The speaker reference used is `1221-135767-0014.wav`, which is unseen during training.\n",
+        "\n",
+        "#### With `embedding_scale=1`\n",
+        "This is the classifier-free guidance scale. The higher the scale, the more conditional the style is to the input text and hence more emotional."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5FwE9CefKQk6"
+      },
+      "outputs": [],
+      "source": [
+        "ref_s = compute_style(\"Demo/reference_audio/1221-135767-0014.wav\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0CKMI0ZsKUDh"
+      },
+      "outputs": [],
+      "source": [
+        "texts = {}\n",
+        "texts['Happy'] = \"We are happy to invite you to join us on a journey to the past, where we will visit the most amazing monuments ever built by human hands.\"\n",
+        "texts['Sad'] = \"I am sorry to say that we have suffered a severe setback in our efforts to restore prosperity and confidence.\"\n",
+        "texts['Angry'] = \"The field of astronomy is a joke! Its theories are based on flawed observations and biased interpretations!\"\n",
+        "texts['Surprised'] = \"I can't believe it! You mean to tell me that you have discovered a new species of bacteria in this pond?\"\n",
+        "\n",
+        "for k,v in texts.items():\n",
+        "    wav = inference(v, ref_s, diffusion_steps=10, alpha=0.3, beta=0.7, embedding_scale=1)\n",
+        "    print(k + \": \")\n",
+        "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "reemQKVEKWAZ"
+      },
+      "source": [
+        "#### With `embedding_scale=2`"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "npIAiAUvKYGv"
+      },
+      "outputs": [],
+      "source": [
+        "texts = {}\n",
+        "texts['Happy'] = \"We are happy to invite you to join us on a journey to the past, where we will visit the most amazing monuments ever built by human hands.\"\n",
+        "texts['Sad'] = \"I am sorry to say that we have suffered a severe setback in our efforts to restore prosperity and confidence.\"\n",
+        "texts['Angry'] = \"The field of astronomy is a joke! Its theories are based on flawed observations and biased interpretations!\"\n",
+        "texts['Surprised'] = \"I can't believe it! You mean to tell me that you have discovered a new species of bacteria in this pond?\"\n",
+        "\n",
+        "for k,v in texts.items():\n",
+        "    noise = torch.randn(1,1,256).to(device)\n",
+        "    wav = inference(v, ref_s, diffusion_steps=10, alpha=0.3, beta=0.7, embedding_scale=2)\n",
+        "    print(k + \": \")\n",
+        "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lqKZaXeYKbrH"
+      },
+      "source": [
+        "#### With `embedding_scale=2, alpha = 0.5, beta = 0.9`\n",
+        "`alpha` and `beta` is the factor to determine much we use the style sampled based on the text instead of the reference. The higher the value of `alpha` and `beta`, the more suitable the style it is to the text but less similar to the reference. Using higher beta makes the synthesized speech more emotional, at the cost of lower similarity to the reference. `alpha` determines the timbre of the speaker while `beta` determines the prosody."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VjXuRCCWKcdN"
+      },
+      "outputs": [],
+      "source": [
+        "texts = {}\n",
+        "texts['Happy'] = \"We are happy to invite you to join us on a journey to the past, where we will visit the most amazing monuments ever built by human hands.\"\n",
+        "texts['Sad'] = \"I am sorry to say that we have suffered a severe setback in our efforts to restore prosperity and confidence.\"\n",
+        "texts['Angry'] = \"The field of astronomy is a joke! Its theories are based on flawed observations and biased interpretations!\"\n",
+        "texts['Surprised'] = \"I can't believe it! You mean to tell me that you have discovered a new species of bacteria in this pond?\"\n",
+        "\n",
+        "for k,v in texts.items():\n",
+        "    noise = torch.randn(1,1,256).to(device)\n",
+        "    wav = inference(v, ref_s, diffusion_steps=10, alpha=0.5, beta=0.9, embedding_scale=2)\n",
+        "    print(k + \": \")\n",
+        "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xrwYXGh0KiIW"
+      },
+      "source": [
+        "### Zero-shot speaker adaptation\n",
+        "This section recreates the \"Acoustic Environment Maintenance\" and \"Speaker’s Emotion Maintenance\" demo in [Section 4](https://styletts2.github.io/#libri) of the demo page. You can compare the generated samples to popular zero-shot TTS models like Vall-E. Note that the model was trained only on LibriTTS, which is about 250 times fewer data compared to those used to trian Vall-E with similar or better effect for these maintainance."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ETUywHHmKimE"
+      },
+      "source": [
+        "#### Acoustic Environment Maintenance\n",
+        "\n",
+        "Since we want to maintain the acoustic environment in the speaker (timbre), we set  `alpha = 0` to make the speaker as close to the reference as possible while only changing the prosody according to the text.  "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yvjBK3syKnZL"
+      },
+      "outputs": [],
+      "source": [
+        "reference_dicts = {}\n",
+        "# format: (path, text)\n",
+        "reference_dicts['3'] = (\"Demo/reference_audio/3.wav\", \"As friends thing I definitely I've got more male friends.\")\n",
+        "reference_dicts['4'] = (\"Demo/reference_audio/4.wav\", \"Everything is run by computer but you got to know how to think before you can do a computer.\")\n",
+        "reference_dicts['5'] = (\"Demo/reference_audio/5.wav\", \"Then out in LA you guys got a whole another ball game within California to worry about.\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jclowWp4KomJ"
+      },
+      "outputs": [],
+      "source": [
+        "noise = torch.randn(1,1,256).to(device)\n",
+        "for k, v in reference_dicts.items():\n",
+        "    path, text = v\n",
+        "    ref_s = compute_style(path)\n",
+        "    start = time.time()\n",
+        "    wav = inference(text, ref_s, alpha=0.0, beta=0.5, diffusion_steps=5, embedding_scale=1)\n",
+        "    rtf = (time.time() - start) / (len(wav) / 24000)\n",
+        "    print(f\"RTF = {rtf:5f}\")\n",
+        "    import IPython.display as ipd\n",
+        "    print('Synthesized: ' + text)\n",
+        "    display(ipd.Audio(wav, rate=24000, normalize=False))\n",
+        "    print('Reference:')\n",
+        "    display(ipd.Audio(path, rate=24000, normalize=False))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LgIm7M93KqVZ"
+      },
+      "source": [
+        "#### Speaker’s Emotion Maintenance\n",
+        "\n",
+        "Since we want to maintain the emotion in the speaker (prosody), we set  `beta = 0.1` to make the speaker as closer to the reference as possible while having some diversity thruogh the slight timbre change."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yzsNoP6oKulL"
+      },
+      "outputs": [],
+      "source": [
+        "reference_dicts = {}\n",
+        "# format: (path, text)\n",
+        "reference_dicts['Anger'] = (\"Demo/reference_audio/anger.wav\", \"We have to reduce the number of plastic bags.\")\n",
+        "reference_dicts['Sleepy'] = (\"Demo/reference_audio/sleepy.wav\", \"We have to reduce the number of plastic bags.\")\n",
+        "reference_dicts['Amused'] = (\"Demo/reference_audio/amused.wav\", \"We have to reduce the number of plastic bags.\")\n",
+        "reference_dicts['Disgusted'] = (\"Demo/reference_audio/disgusted.wav\", \"We have to reduce the number of plastic bags.\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7h2-9cpfKwr4"
+      },
+      "outputs": [],
+      "source": [
+        "noise = torch.randn(1,1,256).to(device)\n",
+        "for k, v in reference_dicts.items():\n",
+        "    path, text = v\n",
+        "    ref_s = compute_style(path)\n",
+        "    start = time.time()\n",
+        "    wav = inference(text, ref_s, alpha=0.3, beta=0.1, diffusion_steps=10, embedding_scale=1)\n",
+        "    rtf = (time.time() - start) / (len(wav) / 24000)\n",
+        "    print(f\"RTF = {rtf:5f}\")\n",
+        "    import IPython.display as ipd\n",
+        "    print(k + ' Synthesized: ' + text)\n",
+        "    display(ipd.Audio(wav, rate=24000, normalize=False))\n",
+        "    print(k + ' Reference:')\n",
+        "    display(ipd.Audio(path, rate=24000, normalize=False))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aNS82PGwKzgg"
+      },
+      "source": [
+        "### Longform Narration\n",
+        "\n",
+        "This section includes basic implementation of Algorithm 1 in the paper for consistent longform audio generation. The example passage is taken from [Section 5](https://styletts2.github.io/#long) of the demo page."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "qs97nL5HK5DH"
+      },
+      "outputs": [],
+      "source": [
+        "passage = passage = '''If the supply of fruit is greater than the family needs, it may be made a source of income by sending the fresh fruit to the market if there is one near enough, or by preserving, canning, and making jelly for sale. To make such an enterprise a success the fruit and work must be first class. There is magic in the word \"Homemade,\" when the product appeals to the eye and the palate; but many careless and incompetent people have found to their sorrow that this word has not magic enough to float inferior goods on the market. As a rule large canning and preserving establishments are clean and have the best appliances, and they employ chemists and skilled labor. The home product must be very good to compete with the attractive goods that are sent out from such establishments. Yet for first class home made products there is a market in all large cities. All first-class grocers have customers who purchase such goods.''' # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "background_save": true
+        },
+        "id": "8Mu9whHYK_1b"
+      },
+      "outputs": [],
+      "source": [
+        "# seen speaker\n",
+        "path = \"Demo/reference_audio/696_92939_000016_000006.wav\"\n",
+        "s_ref = compute_style(path)\n",
+        "sentences = passage.split('.') # simple split by comma\n",
+        "wavs = []\n",
+        "s_prev = None\n",
+        "for text in sentences:\n",
+        "    if text.strip() == \"\": continue\n",
+        "    text += '.' # add it back\n",
+        "\n",
+        "    wav, s_prev = LFinference(text,\n",
+        "                              s_prev,\n",
+        "                              s_ref,\n",
+        "                              alpha = 0.3,\n",
+        "                              beta = 0.9,  # make it more suitable for the text\n",
+        "                              t = 0.7,\n",
+        "                              diffusion_steps=10, embedding_scale=1.5)\n",
+        "    wavs.append(wav)\n",
+        "print('Synthesized: ')\n",
+        "display(ipd.Audio(np.concatenate(wavs), rate=24000, normalize=False))\n",
+        "print('Reference: ')\n",
+        "display(ipd.Audio(path, rate=24000, normalize=False))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "81Rh-lgWLB2i"
+      },
+      "source": [
+        "### Style Transfer\n",
+        "\n",
+        "The following section demostrates the style transfer capacity for unseen speakers in [Section 6](https://styletts2.github.io/#emo) of the demo page. For this, we set `alpha=0.5, beta = 0.9` for the most pronounced effects (mostly using the sampled style)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CtIgr5kOLE9a"
+      },
+      "outputs": [],
+      "source": [
+        "# reference texts to sample styles\n",
+        "\n",
+        "ref_texts = {}\n",
+        "ref_texts['Happy'] = \"We are happy to invite you to join us on a journey to the past, where we will visit the most amazing monuments ever built by human hands.\"\n",
+        "ref_texts['Sad'] = \"I am sorry to say that we have suffered a severe setback in our efforts to restore prosperity and confidence.\"\n",
+        "ref_texts['Angry'] = \"The field of astronomy is a joke! Its theories are based on flawed observations and biased interpretations!\"\n",
+        "ref_texts['Surprised'] = \"I can't believe it! You mean to tell me that you have discovered a new species of bacteria in this pond?\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MlA1CbhzLIoI"
+      },
+      "outputs": [],
+      "source": [
+        "path = \"Demo/reference_audio/1221-135767-0014.wav\"\n",
+        "s_ref = compute_style(path)\n",
+        "\n",
+        "text = \"Yea, his honourable worship is within, but he hath a godly minister or two with him, and likewise a leech.\"\n",
+        "for k,v in texts.items():\n",
+        "    wav = STinference(text, s_ref, v, diffusion_steps=10, alpha=0.5, beta=0.9, embedding_scale=1.5)\n",
+        "    print(k + \": \")\n",
+        "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2M0iaXlkLJUQ"
+      },
+      "source": [
+        "### Speech diversity\n",
+        "\n",
+        "This section reproduces samples in [Section 7](https://styletts2.github.io/#var) of the demo page.\n",
+        "\n",
+        "`alpha` and `beta` determine the diversity of the synthesized speech. There are two extreme cases:\n",
+        "- If `alpha = 1` and `beta = 1`, the synthesized speech sounds the most dissimilar to the reference speaker, but it is also the most diverse (each time you synthesize a speech it will be totally different).\n",
+        "- If `alpha = 0` and `beta = 0`, the synthesized speech sounds the most siimlar to the reference speaker, but it is deterministic (i.e., the sampled style is not used for speech synthesis).\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tSxZDvF2LNu4"
+      },
+      "source": [
+        "#### Default setting (`alpha = 0.3, beta=0.7`)\n",
+        "This setting uses 70% of the reference timbre and 30% of the reference prosody and use the diffusion model to sample them based on the text."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "AAomGCDZLIt5"
+      },
+      "outputs": [],
+      "source": [
+        "# unseen speaker\n",
+        "path = \"Demo/reference_audio/1221-135767-0014.wav\"\n",
+        "ref_s = compute_style(path)\n",
+        "\n",
+        "text = \"How much variation is there?\"\n",
+        "for _ in range(5):\n",
+        "    wav = inference(text, ref_s, diffusion_steps=10, alpha=0.3, beta=0.7, embedding_scale=1)\n",
+        "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BKrSMdgcLQRP"
+      },
+      "source": [
+        "#### Less diverse setting (`alpha = 0.1, beta=0.3`)\n",
+        "This setting uses 90% of the reference timbre and 70% of the reference prosody. This makes it more similar to the reference speaker at cost of less diverse samples."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Uo7gVmFoLRfm"
+      },
+      "outputs": [],
+      "source": [
+        "# unseen speaker\n",
+        "path = \"Demo/reference_audio/1221-135767-0014.wav\"\n",
+        "ref_s = compute_style(path)\n",
+        "\n",
+        "text = \"How much variation is there?\"\n",
+        "for _ in range(5):\n",
+        "    wav = inference(text, ref_s, diffusion_steps=10, alpha=0.1, beta=0.3, embedding_scale=1)\n",
+        "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nfQ0Xrg9LStd"
+      },
+      "source": [
+        "#### More diverse setting (`alpha = 0.5, beta=0.95`)\n",
+        "This setting uses 50% of the reference timbre and 5% of the reference prosody (so it uses 100% of the sampled prosody, which makes it more diverse), but this makes it more dissimilar to the reference speaker.  "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cPHz4BzVLT_u"
+      },
+      "outputs": [],
+      "source": [
+        "# unseen speaker\n",
+        "path = \"Demo/reference_audio/1221-135767-0014.wav\"\n",
+        "ref_s = compute_style(path)\n",
+        "\n",
+        "text = \"How much variation is there?\"\n",
+        "for _ in range(5):\n",
+        "    wav = inference(text, ref_s, diffusion_steps=10, alpha=0.5, beta=0.95, embedding_scale=1)\n",
+        "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "#### Extreme setting (`alpha = 1, beta=1`)\n",
+        "This setting uses 0% of the reference timbre and prosody and use the diffusion model to sample the entire style. This makes the speaker very dissimilar to the reference speaker."
+      ],
+      "metadata": {
+        "id": "hPKg9eYpL00f"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# unseen speaker\n",
+        "path = \"Demo/reference_audio/1221-135767-0014.wav\"\n",
+        "ref_s = compute_style(path)\n",
+        "\n",
+        "text = \"How much variation is there?\"\n",
+        "for _ in range(5):\n",
+        "    wav = inference(text, ref_s, diffusion_steps=10, alpha=1, beta=1, embedding_scale=1)\n",
+        "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+      ],
+      "metadata": {
+        "id": "Ei-7JOccL0bF"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "#### No variation (`alpha = 0, beta=0`)\n",
+        "This setting uses 100% of the reference timbre and prosody and do not use the diffusion model at all. This makes the speaker very similar to the reference speaker, but there is no variation."
+      ],
+      "metadata": {
+        "id": "FVMPc3bhL3eL"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# unseen speaker\n",
+        "path = \"Demo/reference_audio/1221-135767-0014.wav\"\n",
+        "ref_s = compute_style(path)\n",
+        "\n",
+        "text = \"How much variation is there?\"\n",
+        "for _ in range(5):\n",
+        "    wav = inference(text, ref_s, diffusion_steps=10, alpha=0, beta=0, embedding_scale=1)\n",
+        "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+      ],
+      "metadata": {
+        "id": "yh1QZ7uhL4wM"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Extra fun!\n",
+        "\n",
+        "You can record your own voice and clone it using pre-trained StyleTTS 2 model here."
+      ],
+      "metadata": {
+        "id": "T0EvkWrAMBDB"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "#### Run the following cell to record your voice for 5 seconds. Please keep speaking to have the best effect."
+      ],
+      "metadata": {
+        "id": "R985j5QONY8I"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# all imports\n",
+        "from IPython.display import Javascript\n",
+        "from google.colab import output\n",
+        "from base64 import b64decode\n",
+        "\n",
+        "RECORD = \"\"\"\n",
+        "const sleep  = time => new Promise(resolve => setTimeout(resolve, time))\n",
+        "const b2text = blob => new Promise(resolve => {\n",
+        "  const reader = new FileReader()\n",
+        "  reader.onloadend = e => resolve(e.srcElement.result)\n",
+        "  reader.readAsDataURL(blob)\n",
+        "})\n",
+        "var record = time => new Promise(async resolve => {\n",
+        "  stream = await navigator.mediaDevices.getUserMedia({ audio: true })\n",
+        "  recorder = new MediaRecorder(stream)\n",
+        "  chunks = []\n",
+        "  recorder.ondataavailable = e => chunks.push(e.data)\n",
+        "  recorder.start()\n",
+        "  await sleep(time)\n",
+        "  recorder.onstop = async ()=>{\n",
+        "    blob = new Blob(chunks)\n",
+        "    text = await b2text(blob)\n",
+        "    resolve(text)\n",
+        "  }\n",
+        "  recorder.stop()\n",
+        "})\n",
+        "\"\"\"\n",
+        "\n",
+        "def record(sec=3):\n",
+        "  display(Javascript(RECORD))\n",
+        "  s = output.eval_js('record(%d)' % (sec*1000))\n",
+        "  b = b64decode(s.split(',')[1])\n",
+        "  with open('audio.wav','wb') as f:\n",
+        "    f.write(b)\n",
+        "  return 'audio.wav'  # or webm ?"
+      ],
+      "metadata": {
+        "id": "MWrFs0KWMBpz"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "#### Please run this cell and speak:"
+      ],
+      "metadata": {
+        "id": "z35qXwM0Nhx1"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "print('Speak now for 5 seconds.')\n",
+        "audio = record(sec=5)\n",
+        "import IPython.display as ipd\n",
+        "display(ipd.Audio(audio, rate=24000, normalize=False))"
+      ],
+      "metadata": {
+        "id": "KUEoFyQBMR-8"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "#### Synthesize in your own voice"
+      ],
+      "metadata": {
+        "id": "OQS_7IBpNmM1"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "text = ''' StyleTTS 2 is a text to speech model that leverages style diffusion and adversarial training with large speech language models to achieve human level text to speech synthesis. ''' # @param {type:\"string\"}\n"
+      ],
+      "metadata": {
+        "cellView": "form",
+        "id": "c0I3LY7vM8Ta"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "reference_dicts = {}\n",
+        "reference_dicts['You'] = audio"
+      ],
+      "metadata": {
+        "id": "80eW-pwxNCxu"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "start = time.time()\n",
+        "noise = torch.randn(1,1,256).to(device)\n",
+        "for k, path in reference_dicts.items():\n",
+        "    ref_s = compute_style(path)\n",
+        "\n",
+        "    wav = inference(text, ref_s, alpha=0.1, beta=0.5, diffusion_steps=5, embedding_scale=1)\n",
+        "    rtf = (time.time() - start) / (len(wav) / 24000)\n",
+        "    print('Speaker: ' + k)\n",
+        "    import IPython.display as ipd\n",
+        "    print('Synthesized:')\n",
+        "    display(ipd.Audio(wav, rate=24000, normalize=False))\n",
+        "    print('Reference:')\n",
+        "    display(ipd.Audio(path, rate=24000, normalize=False))"
+      ],
+      "metadata": {
+        "id": "yIga6MTuNJaN"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "provenance": [],
+      "collapsed_sections": [
+        "aAGQPfgYIR23",
+        "eJdB_nCOIVIN",
+        "R985j5QONY8I"
+      ],
+      "authorship_tag": "ABX9TyPQdFTqqVEknEG/ma/HMfU+",
+      "include_colab_link": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

Colab/StyleTTS2_Finetune_Demo.ipynb

@@ -0,0 +1,480 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "gpuType": "T4",
+      "authorship_tag": "ABX9TyNiDU9ykIeYxO86Lmuid+ph",
+      "include_colab_link": true
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "accelerator": "GPU"
+  },
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "view-in-github",
+        "colab_type": "text"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/yl4579/StyleTTS2/blob/main/Colab/StyleTTS2_Finetune_Demo.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Install packages and download models"
+      ],
+      "metadata": {
+        "id": "yLqBa4uYPrqE"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "%%shell\n",
+        "git clone https://github.com/yl4579/StyleTTS2.git\n",
+        "cd StyleTTS2\n",
+        "pip install SoundFile torchaudio munch torch pydub pyyaml librosa nltk matplotlib accelerate transformers phonemizer einops einops-exts tqdm typing-extensions git+https://github.com/resemble-ai/monotonic_align.git\n",
+        "sudo apt-get install espeak-ng\n",
+        "git-lfs clone https://huggingface.co/yl4579/StyleTTS2-LibriTTS\n",
+        "mv StyleTTS2-LibriTTS/Models ."
+      ],
+      "metadata": {
+        "id": "H72WF06ZPrTF"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Download dataset (LJSpeech, 200 samples, ~15 minutes of data)\n",
+        "\n",
+        "You can definitely do it with fewer samples. This is just a proof of concept with 200 smaples."
+      ],
+      "metadata": {
+        "id": "G398sL8wPzTB"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "%cd StyleTTS2\n",
+        "!rm -rf Data"
+      ],
+      "metadata": {
+        "id": "kJuQUBrEPy5C"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "!gdown --id 1vqz26D3yn7OXS2vbfYxfSnpLS6m6tOFP\n",
+        "!unzip Data.zip"
+      ],
+      "metadata": {
+        "id": "mDXW8ZZePuSb"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Change the finetuning config\n",
+        "\n",
+        "Depending on the GPU you got, you may want to change the bacth size, max audio length, epiochs and so on."
+      ],
+      "metadata": {
+        "id": "_AlBQREWU8ud"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "config_path = \"Configs/config_ft.yml\"\n",
+        "\n",
+        "import yaml\n",
+        "config = yaml.safe_load(open(config_path))"
+      ],
+      "metadata": {
+        "id": "7uEITi0hU4I2"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "config['data_params']['root_path'] = \"Data/wavs\"\n",
+        "\n",
+        "config['batch_size'] = 2 # not enough RAM\n",
+        "config['max_len'] = 100 # not enough RAM\n",
+        "config['loss_params']['joint_epoch'] = 110 # we do not do SLM adversarial training due to not enough RAM\n",
+        "\n",
+        "with open(config_path, 'w') as outfile:\n",
+        "  yaml.dump(config, outfile, default_flow_style=True)"
+      ],
+      "metadata": {
+        "id": "TPTRgOKSVT4K"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Start finetuning\n"
+      ],
+      "metadata": {
+        "id": "uUuB_19NWj2Y"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "!python train_finetune.py --config_path ./Configs/config_ft.yml"
+      ],
+      "metadata": {
+        "id": "HZVAD5GKWm-O"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Test the model quality\n",
+        "\n",
+        "Note that this mainly serves as a proof of concept due to RAM limitation of free Colab instances. A lot of settings are suboptimal. In the future when DDP works for train_second.py, we will also add mixed precision finetuning to save time and RAM. You can also add SLM adversarial training run if you have paid Colab services (such as A100 with 40G of RAM)."
+      ],
+      "metadata": {
+        "id": "I0_7wsGkXGfc"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import nltk\n",
+        "nltk.download('punkt')"
+      ],
+      "metadata": {
+        "id": "OPLphjbncE7p"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import torch\n",
+        "torch.manual_seed(0)\n",
+        "torch.backends.cudnn.benchmark = False\n",
+        "torch.backends.cudnn.deterministic = True\n",
+        "\n",
+        "import random\n",
+        "random.seed(0)\n",
+        "\n",
+        "import numpy as np\n",
+        "np.random.seed(0)\n",
+        "\n",
+        "# load packages\n",
+        "import time\n",
+        "import random\n",
+        "import yaml\n",
+        "from munch import Munch\n",
+        "import numpy as np\n",
+        "import torch\n",
+        "from torch import nn\n",
+        "import torch.nn.functional as F\n",
+        "import torchaudio\n",
+        "import librosa\n",
+        "from nltk.tokenize import word_tokenize\n",
+        "\n",
+        "from models import *\n",
+        "from utils import *\n",
+        "from text_utils import TextCleaner\n",
+        "textclenaer = TextCleaner()\n",
+        "\n",
+        "%matplotlib inline\n",
+        "\n",
+        "to_mel = torchaudio.transforms.MelSpectrogram(\n",
+        "    n_mels=80, n_fft=2048, win_length=1200, hop_length=300)\n",
+        "mean, std = -4, 4\n",
+        "\n",
+        "def length_to_mask(lengths):\n",
+        "    mask = torch.arange(lengths.max()).unsqueeze(0).expand(lengths.shape[0], -1).type_as(lengths)\n",
+        "    mask = torch.gt(mask+1, lengths.unsqueeze(1))\n",
+        "    return mask\n",
+        "\n",
+        "def preprocess(wave):\n",
+        "    wave_tensor = torch.from_numpy(wave).float()\n",
+        "    mel_tensor = to_mel(wave_tensor)\n",
+        "    mel_tensor = (torch.log(1e-5 + mel_tensor.unsqueeze(0)) - mean) / std\n",
+        "    return mel_tensor\n",
+        "\n",
+        "def compute_style(path):\n",
+        "    wave, sr = librosa.load(path, sr=24000)\n",
+        "    audio, index = librosa.effects.trim(wave, top_db=30)\n",
+        "    if sr != 24000:\n",
+        "        audio = librosa.resample(audio, sr, 24000)\n",
+        "    mel_tensor = preprocess(audio).to(device)\n",
+        "\n",
+        "    with torch.no_grad():\n",
+        "        ref_s = model.style_encoder(mel_tensor.unsqueeze(1))\n",
+        "        ref_p = model.predictor_encoder(mel_tensor.unsqueeze(1))\n",
+        "\n",
+        "    return torch.cat([ref_s, ref_p], dim=1)\n",
+        "\n",
+        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
+        "\n",
+        "# load phonemizer\n",
+        "import phonemizer\n",
+        "global_phonemizer = phonemizer.backend.EspeakBackend(language='en-us', preserve_punctuation=True,  with_stress=True)\n",
+        "\n",
+        "config = yaml.safe_load(open(\"Models/LJSpeech/config_ft.yml\"))\n",
+        "\n",
+        "# load pretrained ASR model\n",
+        "ASR_config = config.get('ASR_config', False)\n",
+        "ASR_path = config.get('ASR_path', False)\n",
+        "text_aligner = load_ASR_models(ASR_path, ASR_config)\n",
+        "\n",
+        "# load pretrained F0 model\n",
+        "F0_path = config.get('F0_path', False)\n",
+        "pitch_extractor = load_F0_models(F0_path)\n",
+        "\n",
+        "# load BERT model\n",
+        "from Utils.PLBERT.util import load_plbert\n",
+        "BERT_path = config.get('PLBERT_dir', False)\n",
+        "plbert = load_plbert(BERT_path)\n",
+        "\n",
+        "model_params = recursive_munch(config['model_params'])\n",
+        "model = build_model(model_params, text_aligner, pitch_extractor, plbert)\n",
+        "_ = [model[key].eval() for key in model]\n",
+        "_ = [model[key].to(device) for key in model]"
+      ],
+      "metadata": {
+        "id": "jIIAoDACXJL0"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "files = [f for f in os.listdir(\"Models/LJSpeech/\") if f.endswith('.pth')]\n",
+        "sorted_files = sorted(files, key=lambda x: int(x.split('_')[-1].split('.')[0]))"
+      ],
+      "metadata": {
+        "id": "eKXRAyyzcMpQ"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "params_whole = torch.load(\"Models/LJSpeech/\" + sorted_files[-1], map_location='cpu')\n",
+        "params = params_whole['net']"
+      ],
+      "metadata": {
+        "id": "ULuU9-VDb9Pk"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "for key in model:\n",
+        "    if key in params:\n",
+        "        print('%s loaded' % key)\n",
+        "        try:\n",
+        "            model[key].load_state_dict(params[key])\n",
+        "        except:\n",
+        "            from collections import OrderedDict\n",
+        "            state_dict = params[key]\n",
+        "            new_state_dict = OrderedDict()\n",
+        "            for k, v in state_dict.items():\n",
+        "                name = k[7:] # remove `module.`\n",
+        "                new_state_dict[name] = v\n",
+        "            # load params\n",
+        "            model[key].load_state_dict(new_state_dict, strict=False)\n",
+        "#             except:\n",
+        "#                 _load(params[key], model[key])\n",
+        "_ = [model[key].eval() for key in model]"
+      ],
+      "metadata": {
+        "id": "J-U29yIYc2ea"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from Modules.diffusion.sampler import DiffusionSampler, ADPM2Sampler, KarrasSchedule"
+      ],
+      "metadata": {
+        "id": "jrPQ_Yrwc3n6"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "sampler = DiffusionSampler(\n",
+        "    model.diffusion.diffusion,\n",
+        "    sampler=ADPM2Sampler(),\n",
+        "    sigma_schedule=KarrasSchedule(sigma_min=0.0001, sigma_max=3.0, rho=9.0), # empirical parameters\n",
+        "    clamp=False\n",
+        ")"
+      ],
+      "metadata": {
+        "id": "n2CWYNoqc455"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "def inference(text, ref_s, alpha = 0.3, beta = 0.7, diffusion_steps=5, embedding_scale=1):\n",
+        "    text = text.strip()\n",
+        "    ps = global_phonemizer.phonemize([text])\n",
+        "    ps = word_tokenize(ps[0])\n",
+        "    ps = ' '.join(ps)\n",
+        "    tokens = textclenaer(ps)\n",
+        "    tokens.insert(0, 0)\n",
+        "    tokens = torch.LongTensor(tokens).to(device).unsqueeze(0)\n",
+        "\n",
+        "    with torch.no_grad():\n",
+        "        input_lengths = torch.LongTensor([tokens.shape[-1]]).to(device)\n",
+        "        text_mask = length_to_mask(input_lengths).to(device)\n",
+        "\n",
+        "        t_en = model.text_encoder(tokens, input_lengths, text_mask)\n",
+        "        bert_dur = model.bert(tokens, attention_mask=(~text_mask).int())\n",
+        "        d_en = model.bert_encoder(bert_dur).transpose(-1, -2)\n",
+        "\n",
+        "        s_pred = sampler(noise = torch.randn((1, 256)).unsqueeze(1).to(device),\n",
+        "                                          embedding=bert_dur,\n",
+        "                                          embedding_scale=embedding_scale,\n",
+        "                                            features=ref_s, # reference from the same speaker as the embedding\n",
+        "                                             num_steps=diffusion_steps).squeeze(1)\n",
+        "\n",
+        "\n",
+        "        s = s_pred[:, 128:]\n",
+        "        ref = s_pred[:, :128]\n",
+        "\n",
+        "        ref = alpha * ref + (1 - alpha)  * ref_s[:, :128]\n",
+        "        s = beta * s + (1 - beta)  * ref_s[:, 128:]\n",
+        "\n",
+        "        d = model.predictor.text_encoder(d_en,\n",
+        "                                         s, input_lengths, text_mask)\n",
+        "\n",
+        "        x, _ = model.predictor.lstm(d)\n",
+        "        duration = model.predictor.duration_proj(x)\n",
+        "\n",
+        "        duration = torch.sigmoid(duration).sum(axis=-1)\n",
+        "        pred_dur = torch.round(duration.squeeze()).clamp(min=1)\n",
+        "\n",
+        "        pred_aln_trg = torch.zeros(input_lengths, int(pred_dur.sum().data))\n",
+        "        c_frame = 0\n",
+        "        for i in range(pred_aln_trg.size(0)):\n",
+        "            pred_aln_trg[i, c_frame:c_frame + int(pred_dur[i].data)] = 1\n",
+        "            c_frame += int(pred_dur[i].data)\n",
+        "\n",
+        "        # encode prosody\n",
+        "        en = (d.transpose(-1, -2) @ pred_aln_trg.unsqueeze(0).to(device))\n",
+        "        if model_params.decoder.type == \"hifigan\":\n",
+        "            asr_new = torch.zeros_like(en)\n",
+        "            asr_new[:, :, 0] = en[:, :, 0]\n",
+        "            asr_new[:, :, 1:] = en[:, :, 0:-1]\n",
+        "            en = asr_new\n",
+        "\n",
+        "        F0_pred, N_pred = model.predictor.F0Ntrain(en, s)\n",
+        "\n",
+        "        asr = (t_en @ pred_aln_trg.unsqueeze(0).to(device))\n",
+        "        if model_params.decoder.type == \"hifigan\":\n",
+        "            asr_new = torch.zeros_like(asr)\n",
+        "            asr_new[:, :, 0] = asr[:, :, 0]\n",
+        "            asr_new[:, :, 1:] = asr[:, :, 0:-1]\n",
+        "            asr = asr_new\n",
+        "\n",
+        "        out = model.decoder(asr,\n",
+        "                                F0_pred, N_pred, ref.squeeze().unsqueeze(0))\n",
+        "\n",
+        "\n",
+        "    return out.squeeze().cpu().numpy()[..., :-50] # weird pulse at the end of the model, need to be fixed later"
+      ],
+      "metadata": {
+        "id": "2x5kVb3nc_eY"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Synthesize speech"
+      ],
+      "metadata": {
+        "id": "O159JnwCc6CC"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "text = '''Maltby and Company would issue warrants on them deliverable to the importer, and the goods were then passed to be stored in neighboring warehouses.\n",
+        "'''"
+      ],
+      "metadata": {
+        "id": "ThciXQ6rc9Eq"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# get a random reference in the training set, note that it doesn't matter which one you use\n",
+        "path = \"Data/wavs/LJ001-0110.wav\"\n",
+        "# this style vector ref_s can be saved as a parameter together with the model weights\n",
+        "ref_s = compute_style(path)"
+      ],
+      "metadata": {
+        "id": "jldPkJyCc83a"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "start = time.time()\n",
+        "wav = inference(text, ref_s, alpha=0.9, beta=0.9, diffusion_steps=10, embedding_scale=1)\n",
+        "rtf = (time.time() - start) / (len(wav) / 24000)\n",
+        "print(f\"RTF = {rtf:5f}\")\n",
+        "import IPython.display as ipd\n",
+        "display(ipd.Audio(wav, rate=24000, normalize=False))"
+      ],
+      "metadata": {
+        "id": "_mIU0jqDdQ-c"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ]
+}

Configs/config.yml

@@ -0,0 +1,116 @@
+log_dir: "Models/LJSpeech"
+first_stage_path: "first_stage.pth"
+save_freq: 2
+log_interval: 10
+device: "cuda"
+epochs_1st: 200 # number of epochs for first stage training (pre-training)
+epochs_2nd: 100 # number of peochs for second stage training (joint training)
+batch_size: 16
+max_len: 400 # maximum number of frames
+pretrained_model: ""
+second_stage_load_pretrained: true # set to true if the pre-trained model is for 2nd stage
+load_only_params: false # set to true if do not want to load epoch numbers and optimizer parameters
+
+F0_path: "Utils/JDC/bst.t7"
+ASR_config: "Utils/ASR/config.yml"
+ASR_path: "Utils/ASR/epoch_00080.pth"
+PLBERT_dir: 'Utils/PLBERT/'
+
+data_params:
+  train_data: "Data/train_list.txt"
+  val_data: "Data/val_list.txt"
+  root_path: "/local/LJSpeech-1.1/wavs"
+  OOD_data: "Data/OOD_texts.txt"
+  min_length: 50 # sample until texts with this size are obtained for OOD texts
+
+preprocess_params:
+  sr: 24000
+  spect_params:
+    n_fft: 2048
+    win_length: 1200
+    hop_length: 300
+
+model_params:
+  multispeaker: false
+
+  dim_in: 64 
+  hidden_dim: 512
+  max_conv_dim: 512
+  n_layer: 3
+  n_mels: 80
+
+  n_token: 178 # number of phoneme tokens
+  max_dur: 50 # maximum duration of a single phoneme
+  style_dim: 128 # style vector size
+  
+  dropout: 0.2
+
+  # config for decoder
+  decoder: 
+      type: 'istftnet' # either hifigan or istftnet
+      resblock_kernel_sizes: [3,7,11]
+      upsample_rates :  [10, 6]
+      upsample_initial_channel: 512
+      resblock_dilation_sizes: [[1,3,5], [1,3,5], [1,3,5]]
+      upsample_kernel_sizes: [20, 12]
+      gen_istft_n_fft: 20
+      gen_istft_hop_size: 5
+      
+  # speech language model config
+  slm:
+      model: 'microsoft/wavlm-base-plus'
+      sr: 16000 # sampling rate of SLM
+      hidden: 768 # hidden size of SLM
+      nlayers: 13 # number of layers of SLM
+      initial_channel: 64 # initial channels of SLM discriminator head
+  
+  # style diffusion model config
+  diffusion:
+    embedding_mask_proba: 0.1
+    # transformer config
+    transformer:
+      num_layers: 3
+      num_heads: 8
+      head_features: 64
+      multiplier: 2
+
+    # diffusion distribution config
+    dist:
+      sigma_data: 0.2 # placeholder for estimate_sigma_data set to false
+      estimate_sigma_data: true # estimate sigma_data from the current batch if set to true
+      mean: -3.0
+      std: 1.0
+  
+loss_params:
+    lambda_mel: 5. # mel reconstruction loss
+    lambda_gen: 1. # generator loss
+    lambda_slm: 1. # slm feature matching loss
+    
+    lambda_mono: 1. # monotonic alignment loss (1st stage, TMA)
+    lambda_s2s: 1. # sequence-to-sequence loss (1st stage, TMA)
+    TMA_epoch: 50 # TMA starting epoch (1st stage)
+
+    lambda_F0: 1. # F0 reconstruction loss (2nd stage)
+    lambda_norm: 1. # norm reconstruction loss (2nd stage)
+    lambda_dur: 1. # duration loss (2nd stage)
+    lambda_ce: 20. # duration predictor probability output CE loss (2nd stage)
+    lambda_sty: 1. # style reconstruction loss (2nd stage)
+    lambda_diff: 1. # score matching loss (2nd stage)
+    
+    diff_epoch: 20 # style diffusion starting epoch (2nd stage)
+    joint_epoch: 50 # joint training starting epoch (2nd stage)
+
+optimizer_params:
+  lr: 0.0001 # general learning rate
+  bert_lr: 0.00001 # learning rate for PLBERT
+  ft_lr: 0.00001 # learning rate for acoustic modules
+  
+slmadv_params:
+  min_len: 400 # minimum length of samples
+  max_len: 500 # maximum length of samples
+  batch_percentage: 0.5 # to prevent out of memory, only use half of the original batch size
+  iter: 10 # update the discriminator every this iterations of generator update
+  thresh: 5 # gradient norm above which the gradient is scaled
+  scale: 0.01 # gradient scaling factor for predictors from SLM discriminators
+  sig: 1.5 # sigma for differentiable duration modeling
+  

Configs/config_ft.yml

@@ -0,0 +1,111 @@
+log_dir: "Models/LJSpeech"
+save_freq: 5
+log_interval: 10
+device: "cuda"
+epochs: 50 # number of finetuning epoch (1 hour of data)
+batch_size: 8
+max_len: 400 # maximum number of frames
+pretrained_model: "Models/LibriTTS/epochs_2nd_00020.pth"
+second_stage_load_pretrained: true # set to true if the pre-trained model is for 2nd stage
+load_only_params: true # set to true if do not want to load epoch numbers and optimizer parameters
+
+F0_path: "Utils/JDC/bst.t7"
+ASR_config: "Utils/ASR/config.yml"
+ASR_path: "Utils/ASR/epoch_00080.pth"
+PLBERT_dir: 'Utils/PLBERT/'
+
+data_params:
+  train_data: "Data/train_list.txt"
+  val_data: "Data/val_list.txt"
+  root_path: "/local/LJSpeech-1.1/wavs"
+  OOD_data: "Data/OOD_texts.txt"
+  min_length: 50 # sample until texts with this size are obtained for OOD texts
+
+preprocess_params:
+  sr: 24000
+  spect_params:
+    n_fft: 2048
+    win_length: 1200
+    hop_length: 300
+
+model_params:
+  multispeaker: true
+
+  dim_in: 64 
+  hidden_dim: 512
+  max_conv_dim: 512
+  n_layer: 3
+  n_mels: 80
+
+  n_token: 178 # number of phoneme tokens
+  max_dur: 50 # maximum duration of a single phoneme
+  style_dim: 128 # style vector size
+  
+  dropout: 0.2
+
+  # config for decoder
+  decoder: 
+      type: 'hifigan' # either hifigan or istftnet
+      resblock_kernel_sizes: [3,7,11]
+      upsample_rates :  [10,5,3,2]
+      upsample_initial_channel: 512
+      resblock_dilation_sizes: [[1,3,5], [1,3,5], [1,3,5]]
+      upsample_kernel_sizes: [20,10,6,4]
+      
+  # speech language model config
+  slm:
+      model: 'microsoft/wavlm-base-plus'
+      sr: 16000 # sampling rate of SLM
+      hidden: 768 # hidden size of SLM
+      nlayers: 13 # number of layers of SLM
+      initial_channel: 64 # initial channels of SLM discriminator head
+  
+  # style diffusion model config
+  diffusion:
+    embedding_mask_proba: 0.1
+    # transformer config
+    transformer:
+      num_layers: 3
+      num_heads: 8
+      head_features: 64
+      multiplier: 2
+
+    # diffusion distribution config
+    dist:
+      sigma_data: 0.2 # placeholder for estimate_sigma_data set to false
+      estimate_sigma_data: true # estimate sigma_data from the current batch if set to true
+      mean: -3.0
+      std: 1.0
+  
+loss_params:
+    lambda_mel: 5. # mel reconstruction loss
+    lambda_gen: 1. # generator loss
+    lambda_slm: 1. # slm feature matching loss
+    
+    lambda_mono: 1. # monotonic alignment loss (TMA)
+    lambda_s2s: 1. # sequence-to-sequence loss (TMA)
+
+    lambda_F0: 1. # F0 reconstruction loss
+    lambda_norm: 1. # norm reconstruction loss
+    lambda_dur: 1. # duration loss
+    lambda_ce: 20. # duration predictor probability output CE loss
+    lambda_sty: 1. # style reconstruction loss
+    lambda_diff: 1. # score matching loss
+    
+    diff_epoch: 10 # style diffusion starting epoch
+    joint_epoch: 30 # joint training starting epoch
+
+optimizer_params:
+  lr: 0.0001 # general learning rate
+  bert_lr: 0.00001 # learning rate for PLBERT
+  ft_lr: 0.0001 # learning rate for acoustic modules
+  
+slmadv_params:
+  min_len: 400 # minimum length of samples
+  max_len: 500 # maximum length of samples
+  batch_percentage: 0.5 # to prevent out of memory, only use half of the original batch size
+  iter: 10 # update the discriminator every this iterations of generator update
+  thresh: 5 # gradient norm above which the gradient is scaled
+  scale: 0.01 # gradient scaling factor for predictors from SLM discriminators
+  sig: 1.5 # sigma for differentiable duration modeling
+  

Configs/config_libritts.yml

@@ -0,0 +1,113 @@
+log_dir: "Models/LibriTTS"
+first_stage_path: "first_stage.pth"
+save_freq: 1
+log_interval: 10
+device: "cuda"
+epochs_1st: 50 # number of epochs for first stage training (pre-training)
+epochs_2nd: 30 # number of peochs for second stage training (joint training)
+batch_size: 16
+max_len: 300 # maximum number of frames
+pretrained_model: ""
+second_stage_load_pretrained: true # set to true if the pre-trained model is for 2nd stage
+load_only_params: false # set to true if do not want to load epoch numbers and optimizer parameters
+
+F0_path: "Utils/JDC/bst.t7"
+ASR_config: "Utils/ASR/config.yml"
+ASR_path: "Utils/ASR/epoch_00080.pth"
+PLBERT_dir: 'Utils/PLBERT/'
+
+data_params:
+  train_data: "Data/train_list.txt"
+  val_data: "Data/val_list.txt"
+  root_path: ""
+  OOD_data: "Data/OOD_texts.txt"
+  min_length: 50 # sample until texts with this size are obtained for OOD texts
+
+preprocess_params:
+  sr: 24000
+  spect_params:
+    n_fft: 2048
+    win_length: 1200
+    hop_length: 300
+
+model_params:
+  multispeaker: true
+
+  dim_in: 64 
+  hidden_dim: 512
+  max_conv_dim: 512
+  n_layer: 3
+  n_mels: 80
+
+  n_token: 178 # number of phoneme tokens
+  max_dur: 50 # maximum duration of a single phoneme
+  style_dim: 128 # style vector size
+  
+  dropout: 0.2
+
+  # config for decoder
+  decoder: 
+      type: 'hifigan' # either hifigan or istftnet
+      resblock_kernel_sizes: [3,7,11]
+      upsample_rates :  [10,5,3,2]
+      upsample_initial_channel: 512
+      resblock_dilation_sizes: [[1,3,5], [1,3,5], [1,3,5]]
+      upsample_kernel_sizes: [20,10,6,4]
+      
+  # speech language model config
+  slm:
+      model: 'microsoft/wavlm-base-plus'
+      sr: 16000 # sampling rate of SLM
+      hidden: 768 # hidden size of SLM
+      nlayers: 13 # number of layers of SLM
+      initial_channel: 64 # initial channels of SLM discriminator head
+  
+  # style diffusion model config
+  diffusion:
+    embedding_mask_proba: 0.1
+    # transformer config
+    transformer:
+      num_layers: 3
+      num_heads: 8
+      head_features: 64
+      multiplier: 2
+
+    # diffusion distribution config
+    dist:
+      sigma_data: 0.2 # placeholder for estimate_sigma_data set to false
+      estimate_sigma_data: true # estimate sigma_data from the current batch if set to true
+      mean: -3.0
+      std: 1.0
+  
+loss_params:
+    lambda_mel: 5. # mel reconstruction loss
+    lambda_gen: 1. # generator loss
+    lambda_slm: 1. # slm feature matching loss
+    
+    lambda_mono: 1. # monotonic alignment loss (1st stage, TMA)
+    lambda_s2s: 1. # sequence-to-sequence loss (1st stage, TMA)
+    TMA_epoch: 5 # TMA starting epoch (1st stage)
+
+    lambda_F0: 1. # F0 reconstruction loss (2nd stage)
+    lambda_norm: 1. # norm reconstruction loss (2nd stage)
+    lambda_dur: 1. # duration loss (2nd stage)
+    lambda_ce: 20. # duration predictor probability output CE loss (2nd stage)
+    lambda_sty: 1. # style reconstruction loss (2nd stage)
+    lambda_diff: 1. # score matching loss (2nd stage)
+    
+    diff_epoch: 10 # style diffusion starting epoch (2nd stage)
+    joint_epoch: 15 # joint training starting epoch (2nd stage)
+
+optimizer_params:
+  lr: 0.0001 # general learning rate
+  bert_lr: 0.00001 # learning rate for PLBERT
+  ft_lr: 0.00001 # learning rate for acoustic modules
+  
+slmadv_params:
+  min_len: 400 # minimum length of samples
+  max_len: 500 # maximum length of samples
+  batch_percentage: 0.5 # to prevent out of memory, only use half of the original batch size
+  iter: 20 # update the discriminator every this iterations of generator update
+  thresh: 5 # gradient norm above which the gradient is scaled
+  scale: 0.01 # gradient scaling factor for predictors from SLM discriminators
+  sig: 1.5 # sigma for differentiable duration modeling

Data/OOD_texts.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0989ef6a9873b711befefcbe60660ced7a65532359277f766f4db504c558a72
+size 31758898

Data/val_list.txt

@@ -0,0 +1,100 @@
+LJ022-0023.wav|ðɪ ˌoʊvɚwˈɛlmɪŋ mədʒˈɔːɹᵻɾi ʌv pˈiːpəl ɪn ðɪs kˈʌntɹi nˈoʊ hˌaʊ tə sˈɪft ðə wˈiːt fɹʌmðə tʃˈæf ɪn wʌt ðeɪ hˈɪɹ ænd wʌt ðeɪ ɹˈiːd .|0
+LJ043-0030.wav|ɪf sˈʌmbɑːdi dˈɪd ðˈæt tə mˌiː , ɐ lˈaʊsi tɹˈɪk lˈaɪk ðˈæt , tə tˈeɪk maɪ wˈaɪf ɐwˈeɪ , ænd ˈɔːl ðə fˈɜːnɪtʃɚ , aɪ wʊd biː mˈæd æz hˈɛl , tˈuː .|0
+LJ005-0201.wav|ˌæzˌɪz ʃˈoʊn baɪ ðə ɹᵻpˈoːɹt ʌvðə kəmˈɪʃənɚz tʊ ɪŋkwˈaɪɚɹ ˌɪntʊ ðə stˈeɪt ʌvðə mjuːnˈɪsɪpəl kˌɔːɹpɚɹˈeɪʃənz ɪn ˈeɪtiːn θˈɜːɾi fˈaɪv .|0
+LJ001-0110.wav|ˈiːvən ðə kˈæslɑːn tˈaɪp wɛn ɛnlˈɑːɹdʒd ʃˈoʊz ɡɹˈeɪt ʃˈɔːɹtkʌmɪŋz ɪn ðɪs ɹᵻspˈɛkt :|0
+LJ003-0345.wav|ˈɔːl ðə kəmˈɪɾi kʊd dˈuː ɪn ðɪs ɹᵻspˈɛkt wʌz tə θɹˈoʊ ðə ɹᵻspˌɑːnsəbˈɪlɪɾi ˌɔn ˈʌðɚz .|0
+LJ007-0154.wav|ðiːz pˈʌndʒənt ænd wˈɛl ɡɹˈaʊndᵻd stɹˈɪktʃɚz ɐplˈaɪd wɪð stˈɪl ɡɹˈeɪɾɚ fˈoːɹs tə ðɪ ʌŋkənvˈɪktᵻd pɹˈɪzənɚ , ðə mˈæn hˌuː kˈeɪm tə ðə pɹˈɪzən ˈɪnəsənt , ænd stˈɪl ʌŋkəntˈæmᵻnˌeɪɾᵻd ,|0
+LJ018-0098.wav|ænd ɹˈɛkəɡnˌaɪzd æz wˈʌn ʌvðə fɹˈiːkwɛntɚz ʌvðə bˈoʊɡəs lˈɔː stˈeɪʃənɚz . hɪz ɚɹˈɛst lˈɛd tə ðæt ʌv ˈʌðɚz .|0
+LJ047-0044.wav|ˈɑːswəld wʌz , haʊˈɛvɚ , wˈɪlɪŋ tə dɪskˈʌs hɪz kˈɑːntækts wɪð sˈoʊviət ɐθˈɔːɹɪɾiz . hiː dᵻnˈaɪd hˌævɪŋ ˌɛni ɪnvˈɑːlvmənt wɪð sˈoʊviət ɪntˈɛlɪdʒəns ˈeɪdʒənsiz|0
+LJ031-0038.wav|ðə fˈɜːst fɪzˈɪʃən tə sˈiː ðə pɹˈɛzɪdənt æt pˈɑːɹklənd hˈɑːspɪɾəl wʌz dˈɑːktɚ . tʃˈɑːɹlz dʒˈeɪ . kˈæɹɪkˌoʊ , ɐ ɹˈɛzᵻdənt ɪn dʒˈɛnɚɹəl sˈɜːdʒɚɹi .|0
+LJ048-0194.wav|dˈʊɹɹɪŋ ðə mˈɔːɹnɪŋ ʌv noʊvˈɛmbɚ twˈɛnti tˈuː pɹˈaɪɚ tə ðə mˈoʊɾɚkˌeɪd .|0
+LJ049-0026.wav|ˌɔn əkˈeɪʒən ðə sˈiːkɹᵻt sˈɜːvɪs hɐzbɪn pɚmˈɪɾᵻd tə hæv ɐn ˈeɪdʒənt ɹˈaɪdɪŋ ɪnðə pˈæsɪndʒɚ kəmpˈɑːɹtmənt wɪððə pɹˈɛzɪdənt .|0
+LJ004-0152.wav|ɔːlðˈoʊ æt mˈɪstɚ . bˈʌkstənz vˈɪzɪt ɐ nˈuː dʒˈeɪl wʌz ɪn pɹˈɑːsɛs ʌv ɪɹˈɛkʃən , ðə fˈɜːst stˈɛp təwˈɔːɹdz ɹᵻfˈɔːɹm sˈɪns hˈaʊɚdz vˌɪzɪtˈeɪʃən ɪn sˈɛvəntˌiːn sˈɛvənti fˈoːɹ .|0
+LJ008-0278.wav|ɔːɹ ðˈɛɹz mˌaɪt biː wˈʌn ʌv mˈɛni , ænd ɪt mˌaɪt biː kənsˈɪdɚd nˈɛsᵻsɚɹi tə dˈɑːlɚ mˌeɪk ɐn ɛɡzˈæmpəl.dˈɑːlɚ|0
+LJ043-0002.wav|ðə wˈɔːɹəŋ kəmˈɪʃən ɹᵻpˈoːɹt . baɪ ðə pɹˈɛzɪdənts kəmˈɪʃən ɔnðɪ ɐsˌæsᵻnˈeɪʃən ʌv pɹˈɛzɪdənt kˈɛnədi . tʃˈæptɚ sˈɛvən . lˈiː hˈɑːɹvi ˈɑːswəld :|0
+LJ009-0114.wav|mˈɪstɚ . wˈeɪkfiːld wˈaɪndz ˈʌp hɪz ɡɹˈæfɪk bˌʌt sˈʌmwʌt sɛnsˈeɪʃənəl ɐkˈaʊnt baɪ dᵻskɹˈaɪbɪŋ ɐnˈʌðɚ ɹᵻlˈɪdʒəs sˈɜːvɪs , wˌɪtʃ mˈeɪ ɐpɹˈoʊpɹɪˌeɪtli biː ɪnsˈɜːɾᵻd hˈɪɹ .|0
+LJ028-0506.wav|ɐ mˈɑːdɚn ˈɑːɹɾɪst wʊdhɐv dˈɪfɪkˌʌlti ɪn dˌuːɪŋ sˈʌtʃ ˈækjʊɹət wˈɜːk .|0
+LJ050-0168.wav|wɪððə pɚtˈɪkjʊlɚ pˈɜːpəsᵻz ʌvðɪ ˈeɪdʒənsi ɪnvˈɑːlvd . ðə kəmˈɪʃən ɹˈɛkəɡnˌaɪzᵻz ðæt ðɪs ɪz ɐ kˌɑːntɹəvˈɜːʃəl ˈɛɹiə|0
+LJ039-0223.wav|ˈɑːswəldz mɚɹˈiːn tɹˈeɪnɪŋ ɪn mˈɑːɹksmənʃˌɪp , hɪz ˈʌðɚ ɹˈaɪfəl ɛkspˈiəɹɪəns ænd hɪz ɪstˈæblɪʃt fəmˌɪliˈæɹɪɾi wɪð ðɪs pɚtˈɪkjʊlɚ wˈɛpən|0
+LJ029-0032.wav|ɐkˈoːɹdɪŋ tʊ oʊdˈɑːnəl , kwˈoʊt , wiː hæd ɐ mˈoʊɾɚkˌeɪd wɛɹˈɛvɚ kplˈʌsplʌs wˌɪtʃ hɐdbɪn bˌɪn hˈeɪstili sˈʌmənd fɚðə ðə pˈɜːpəs wiː wˈɛnt , ˈɛnd kwˈoʊt .|0
+LJ031-0070.wav|dˈɑːktɚ . klˈɑːɹk , hˌuː mˈoʊst klˈoʊsli əbzˈɜːvd ðə hˈɛd wˈuːnd ,|0
+LJ034-0198.wav|jˈuːɪnz , hˌuː wʌz ɔnðə saʊθwˈɛst kˈɔːɹnɚɹ ʌv ˈɛlm ænd hjˈuːstən stɹˈiːts tˈɛstᵻfˌaɪd ðæt hiː kʊd nˌɑːt dᵻskɹˈaɪb ðə mˈæn hiː sˈɔː ɪnðə wˈɪndoʊ .|0
+LJ026-0068.wav|ˈɛnɚdʒi ˈɛntɚz ðə plˈænt , tʊ ɐ smˈɔːl ɛkstˈɛnt ,|0
+LJ039-0075.wav|wˈʌns juː nˈoʊ ðæt juː mˈʌst pˌʊt ðə kɹˈɔshɛɹz ɔnðə tˈɑːɹɡɪt ænd ðæt ɪz ˈɔːl ðæt ɪz nˈɛsᵻsɚɹi .|0
+LJ004-0096.wav|ðə fˈeɪɾəl kˈɑːnsɪkwənsᵻz wˈɛɹɑːf mˌaɪt biː pɹɪvˈɛntᵻd ɪf ðə dʒˈʌstɪsᵻz ʌvðə pˈiːs wɜː djˈuːli ˈɔːθɚɹˌaɪzd|0
+LJ005-0014.wav|spˈiːkɪŋ ˌɔn ɐ dᵻbˈeɪt ˌɔn pɹˈɪzən mˈæɾɚz , hiː dᵻklˈɛɹd ðˈæt|0
+LJ012-0161.wav|hiː wʌz ɹᵻpˈoːɹɾᵻd tə hæv fˈɔːlən ɐwˈeɪ tʊ ɐ ʃˈædoʊ .|0
+LJ018-0239.wav|hɪz dˌɪsɐpˈɪɹəns ɡˈeɪv kˈʌlɚ ænd sˈʌbstəns tʊ ˈiːvəl ɹᵻpˈoːɹts ɔːlɹˌɛdi ɪn sˌɜːkjʊlˈeɪʃən ðætðə wɪl ænd kənvˈeɪəns əbˌʌv ɹᵻfˈɜːd tuː|0
+LJ019-0257.wav|hˈɪɹ ðə tɹˈɛd wˈiːl wʌz ɪn jˈuːs , ðɛɹ sˈɛljʊlɚ kɹˈæŋks , ɔːɹ hˈɑːɹd lˈeɪbɚ məʃˈiːnz .|0
+LJ028-0008.wav|juː tˈæp dʒˈɛntli wɪð jʊɹ hˈiːl əpˌɑːn ðə ʃˈoʊldɚɹ ʌvðə dɹˈoʊmdɚɹi tʊ ˈɜːdʒ hɜːɹ ˈɔn .|0
+LJ024-0083.wav|ðɪs plˈæn ʌv mˈaɪn ɪz nˈoʊ ɐtˈæk ɔnðə kˈoːɹt ;|0
+LJ042-0129.wav|nˈoʊ nˈaɪt klˈʌbz ɔːɹ bˈoʊlɪŋ ˈælɪz , nˈoʊ plˈeɪsᵻz ʌv ɹˌɛkɹiːˈeɪʃən ɛksˈɛpt ðə tɹˈeɪd jˈuːniən dˈænsᵻz . aɪ hæv hæd ɪnˈʌf .|0
+LJ036-0103.wav|ðə pəlˈiːs ˈæskt hˌɪm wˈɛðɚ hiː kʊd pˈɪk ˈaʊt hɪz pˈæsɪndʒɚ fɹʌmðə lˈaɪnʌp .|0
+LJ046-0058.wav|dˈʊɹɹɪŋ hɪz pɹˈɛzɪdənsi , fɹˈæŋklɪn dˈiː . ɹˈoʊzəvˌɛlt mˌeɪd ˈɔːlmoʊst fˈoːɹ hˈʌndɹɪd dʒˈɜːniz ænd tɹˈævəld mˈoːɹ ðɐn θɹˈiː hˈʌndɹɪd fˈɪfti θˈaʊzənd mˈaɪlz .|0
+LJ014-0076.wav|hiː wʌz sˈiːn ˈæftɚwɚdz smˈoʊkɪŋ ænd tˈɔːkɪŋ wɪð hɪz hˈoʊsts ɪn ðɛɹ bˈæk pˈɑːɹlɚ , ænd nˈɛvɚ sˈiːn ɐɡˈɛn ɐlˈaɪv .|0
+LJ002-0043.wav|lˈɔŋ nˈæɹoʊ ɹˈuːmz wˈʌn θˈɜːɾi sˈɪks fˈiːt , sˈɪks twˈɛnti θɹˈiː fˈiːt , ænd ðɪ ˈeɪtθ ˈeɪtiːn ,|0
+LJ009-0076.wav|wiː kˈʌm tə ðə sˈɜːmən .|0
+LJ017-0131.wav|ˈiːvən wɛn ðə hˈaɪ ʃˈɛɹɪf hæd tˈoʊld hˌɪm ðɛɹwˌʌz nˈoʊ pˌɑːsəbˈɪlɪɾi əvɚ ɹᵻpɹˈiːv , ænd wɪðˌɪn ɐ fjˈuː ˈaʊɚz ʌv ˌɛksɪkjˈuːʃən .|0
+LJ046-0184.wav|bˌʌt ðɛɹ ɪz ɐ sˈɪstəm fɚðɪ ɪmˈiːdɪət nˌoʊɾɪfɪkˈeɪʃən ʌvðə sˈiːkɹᵻt sˈɜːvɪs baɪ ðə kənfˈaɪnɪŋ ˌɪnstɪtˈuːʃən wɛn ɐ sˈʌbdʒɛkt ɪz ɹᵻlˈiːst ɔːɹ ɛskˈeɪps .|0
+LJ014-0263.wav|wˌɛn ˈʌðɚ plˈɛʒɚz pˈɔːld hiː tˈʊk ɐ θˈiəɾɚ , ænd pˈoʊzd æz ɐ mjuːnˈɪfɪsənt pˈeɪtɹən ʌvðə dɹəmˈæɾɪk ˈɑːɹt .|0
+LJ042-0096.wav|ˈoʊld ɛkstʃˈeɪndʒ ɹˈeɪt ɪn ɐdˈɪʃən tə hɪz fˈæktɚɹi sˈælɚɹi ʌv ɐpɹˈɑːksɪmətli ˈiːkwəl ɐmˈaʊnt|0
+LJ049-0050.wav|hˈɪl hæd bˈoʊθ fˈiːt ɔnðə kˈɑːɹ ænd wʌz klˈaɪmɪŋ ɐbˈoːɹd tʊ ɐsˈɪst pɹˈɛzɪdənt ænd mˈɪsɪz . kˈɛnədi .|0
+LJ019-0186.wav|sˈiːɪŋ ðæt sˈɪns ðɪ ɪstˈæblɪʃmənt ʌvðə sˈɛntɹəl kɹˈɪmɪnəl kˈoːɹt , nˈuːɡeɪt ɹᵻsˈiːvd pɹˈɪzənɚz fɔːɹ tɹˈaɪəl fɹʌm sˈɛvɹəl kˈaʊntiz ,|0
+LJ028-0307.wav|ðˈɛn lˈɛt twˈɛnti dˈeɪz pˈæs , ænd æt ðɪ ˈɛnd ʌv ðæt tˈaɪm stˈeɪʃən nˌɪɹ ðə tʃˈældæsəŋ ɡˈeɪts ɐ bˈɑːdi ʌv fˈoːɹ θˈaʊzənd .|0
+LJ012-0235.wav|wˌaɪl ðeɪ wɜːɹ ɪn ɐ stˈeɪt ʌv ɪnsˌɛnsəbˈɪlɪɾi ðə mˈɜːdɚ wʌz kəmˈɪɾᵻd .|0
+LJ034-0053.wav|ɹˈiːtʃt ðə sˈeɪm kəŋklˈuːʒən æz lætˈoʊnə ðætðə pɹˈɪnts fˈaʊnd ɔnðə kˈɑːɹtənz wɜː ðoʊz ʌv lˈiː hˈɑːɹvi ˈɑːswəld .|0
+LJ014-0030.wav|ðiːz wɜː dˈæmnətˌoːɹi fˈækts wˌɪtʃ wˈɛl səpˈoːɹɾᵻd ðə pɹˌɑːsɪkjˈuːʃən .|0
+LJ015-0203.wav|bˌʌt wɜː ðə pɹɪkˈɔːʃənz tˈuː mˈɪnɪt , ðə vˈɪdʒɪləns tˈuː klˈoʊs təbi ᵻlˈuːdᵻd ɔːɹ ˌoʊvɚkˈʌm ?|0
+LJ028-0093.wav|bˌʌt hɪz skɹˈaɪb ɹˈoʊt ɪɾ ɪnðə mˈænɚ kˈʌstəmˌɛɹi fɚðə skɹˈaɪbz ʌv ðoʊz dˈeɪz tə ɹˈaɪt ʌv ðɛɹ ɹˈɔɪəl mˈæstɚz .|0
+LJ002-0018.wav|ðɪ ɪnˈædɪkwəsi ʌvðə dʒˈeɪl wʌz nˈoʊɾɪst ænd ɹᵻpˈoːɹɾᵻd əpˌɑːn ɐɡˈɛn ænd ɐɡˈɛn baɪ ðə ɡɹˈænd dʒˈʊɹɹiz ʌvðə sˈɪɾi ʌv lˈʌndən ,|0
+LJ028-0275.wav|æt lˈæst , ɪnðə twˈɛntiəθ mˈʌnθ ,|0
+LJ012-0042.wav|wˌɪtʃ hiː kˈɛpt kənsˈiːld ɪn ɐ hˈaɪdɪŋ plˈeɪs wɪð ɐ tɹˈæp dˈoːɹ dʒˈʌst ˌʌndɚ hɪz bˈɛd .|0
+LJ011-0096.wav|hiː mˈæɹid ɐ lˈeɪdi ˈɔːlsoʊ bᵻlˈɔŋɪŋ tə ðə səsˈaɪəɾi ʌv fɹˈɛndz , hˌuː bɹˈɔːt hˌɪm ɐ lˈɑːɹdʒ fˈɔːɹtʃʊn , wˈɪtʃ , ænd hɪz ˈoʊn mˈʌni , hiː pˌʊt ˌɪntʊ ɐ sˈɪɾi fˈɜːm ,|0
+LJ036-0077.wav|ɹˈɑːdʒɚ dˈiː . kɹˈeɪɡ , ɐ dˈɛpjuːɾi ʃˈɛɹɪf ʌv dˈæləs kˈaʊnti ,|0
+LJ016-0318.wav|ˈʌðɚɹ əfˈɪʃəlz , ɡɹˈeɪt lˈɔɪɚz , ɡˈʌvɚnɚz ʌv pɹˈɪzənz , ænd tʃˈæplɪnz səpˈoːɹɾᵻd ðɪs vjˈuː .|0
+LJ013-0164.wav|hˌuː kˈeɪm fɹʌm hɪz ɹˈuːm ɹˈɛdi dɹˈɛst , ɐ səspˈɪʃəs sˈɜːkəmstˌæns , æz hiː wʌz ˈɔːlweɪz lˈeɪt ɪnðə mˈɔːɹnɪŋ .|0
+LJ027-0141.wav|ɪz klˈoʊsli ɹᵻpɹədˈuːst ɪnðə lˈaɪf hˈɪstɚɹi ʌv ɛɡzˈɪstɪŋ dˈɪɹ . ɔːɹ , ɪn ˈʌðɚ wˈɜːdz ,|0
+LJ028-0335.wav|ɐkˈoːɹdɪŋli ðeɪ kəmˈɪɾᵻd tə hˌɪm ðə kəmˈænd ʌv ðɛɹ hˈoʊl ˈɑːɹmi , ænd pˌʊt ðə kˈiːz ʌv ðɛɹ sˈɪɾi ˌɪntʊ hɪz hˈændz .|0
+LJ031-0202.wav|mˈɪsɪz . kˈɛnədi tʃˈoʊz ðə hˈɑːspɪɾəl ɪn bəθˈɛzdə fɚðɪ ˈɔːtɑːpsi bɪkˈʌz ðə pɹˈɛzɪdənt hæd sˈɜːvd ɪnðə nˈeɪvi .|0
+LJ021-0145.wav|fɹʌm ðoʊz wˈɪlɪŋ tə dʒˈɔɪn ɪn ɪstˈæblɪʃɪŋ ðɪs hˈo��pt fɔːɹ pˈiəɹɪəd ʌv pˈiːs ,|0
+LJ016-0288.wav|dˈɑːlɚ mˈuːlɚ , mˈuːlɚ , hiːz ðə mˈæn , dˈɑːlɚ tˈɪl ɐ daɪvˈɜːʒən wʌz kɹiːˈeɪɾᵻd baɪ ðɪ ɐpˈɪɹəns ʌvðə ɡˈæloʊz , wˌɪtʃ wʌz ɹᵻsˈiːvd wɪð kəntˈɪnjuːəs jˈɛlz .|0
+LJ028-0081.wav|jˈɪɹz lˈeɪɾɚ , wˌɛn ðɪ ˌɑːɹkiːˈɑːlədʒˌɪsts kʊd ɹˈɛdili dɪstˈɪŋɡwɪʃ ðə fˈɔls fɹʌmðə tɹˈuː ,|0
+LJ018-0081.wav|hɪz dᵻfˈɛns bˌiːɪŋ ðæt hiː hæd ɪntˈɛndᵻd tə kəmˈɪt sˈuːɪsˌaɪd , bˌʌt ðˈæt , ɔnðɪ ɐpˈɪɹəns ʌv ðɪs ˈɑːfɪsɚ hˌuː hæd ɹˈɔŋd hˌɪm ,|0
+LJ021-0066.wav|təɡˌɛðɚ wɪð ɐ ɡɹˈeɪt ˈɪŋkɹiːs ɪnðə pˈeɪɹoʊlz , ðɛɹ hɐz kˈʌm ɐ səbstˈænʃəl ɹˈaɪz ɪnðə tˈoʊɾəl ʌv ɪndˈʌstɹɪəl pɹˈɑːfɪts|0
+LJ009-0238.wav|ˈæftɚ ðɪs ðə ʃˈɛɹɪfs sˈɛnt fɔːɹ ɐnˈʌðɚ ɹˈoʊp , bˌʌt ðə spɛktˈeɪɾɚz ˌɪntəfˈɪɹd , ænd ðə mˈæn wʌz kˈæɹid bˈæk tə dʒˈeɪl .|0
+LJ005-0079.wav|ænd ɪmpɹˈuːv ðə mˈɔːɹəlz ʌvðə pɹˈɪzənɚz , ænd ʃˌæl ɪnʃˈʊɹ ðə pɹˈɑːpɚ mˈɛʒɚɹ ʌv pˈʌnɪʃmənt tə kənvˈɪktᵻd əfˈɛndɚz .|0
+LJ035-0019.wav|dɹˈoʊv tə ðə nɔːɹθwˈɛst kˈɔːɹnɚɹ ʌv ˈɛlm ænd hjˈuːstən , ænd pˈɑːɹkt ɐpɹˈɑːksɪmətli tˈɛn fˈiːt fɹʌmðə tɹˈæfɪk sˈɪɡnəl .|0
+LJ036-0174.wav|ðɪs ɪz ðɪ ɐpɹˈɑːksɪmət tˈaɪm hiː ˈɛntɚd ðə ɹˈuːmɪŋhˌaʊs , ɐkˈoːɹdɪŋ tʊ ˈɜːliːn ɹˈɑːbɚts , ðə hˈaʊskiːpɚ ðˈɛɹ .|0
+LJ046-0146.wav|ðə kɹaɪtˈiəɹɪə ɪn ɪfˈɛkt pɹˈaɪɚ tə noʊvˈɛmbɚ twˈɛnti tˈuː , nˈaɪntiːn sˈɪksti θɹˈiː , fɔːɹ dɪtˈɜːmɪnɪŋ wˈɛðɚ tʊ ɐksˈɛpt mətˈɪɹiəl fɚðə pˌiːˌɑːɹɹˈɛs dʒˈɛnɚɹəl fˈaɪlz|0
+LJ017-0044.wav|ænd ðə dˈiːpɪst æŋzˈaɪəɾi wʌz fˈɛlt ðætðə kɹˈaɪm , ɪf kɹˈaɪm ðˈɛɹ hɐdbɪn , ʃˌʊd biː bɹˈɔːt hˈoʊm tʊ ɪts pˈɜːpɪtɹˌeɪɾɚ .|0
+LJ017-0070.wav|bˌʌt hɪz spˈoːɹɾɪŋ ˌɑːpɚɹˈeɪʃənz dɪdnˌɑːt pɹˈɑːspɚ , ænd hiː bɪkˌeɪm ɐ nˈiːdi mˈæn , ˈɔːlweɪz dɹˈɪvən tə dˈɛspɚɹət stɹˈeɪts fɔːɹ kˈæʃ .|0
+LJ014-0020.wav|hiː wʌz sˈuːn ˈæftɚwɚdz ɚɹˈɛstᵻd ˌɔn səspˈɪʃən , ænd ɐ sˈɜːtʃ ʌv hɪz lˈɑːdʒɪŋz bɹˈɔːt tə lˈaɪt sˈɛvɹəl ɡˈɑːɹmənts sˈætʃɚɹˌeɪɾᵻd wɪð blˈʌd ;|0
+LJ016-0020.wav|hiː nˈɛvɚ ɹˈiːtʃt ðə sˈɪstɚn , bˌʌt fˈɛl bˈæk ˌɪntʊ ðə jˈɑːɹd , ˈɪndʒɚɹɪŋ hɪz lˈɛɡz sᵻvˈɪɹli .|0
+LJ045-0230.wav|wˌɛn hiː wʌz fˈaɪnəli ˌæpɹihˈɛndᵻd ɪnðə tˈɛksəs θˈiəɾɚ . ɔːlðˈoʊ ɪɾ ɪz nˌɑːt fˈʊli kɚɹˈɑːbɚɹˌeɪɾᵻd baɪ ˈʌðɚz hˌuː wɜː pɹˈɛzənt ,|0
+LJ035-0129.wav|ænd ʃiː mˈʌstɐv ɹˈʌn dˌaʊn ðə stˈɛɹz ɐhˈɛd ʌv ˈɑːswəld ænd wʊd pɹˈɑːbəbli hæv sˈiːn ɔːɹ hˈɜːd hˌɪm .|0
+LJ008-0307.wav|ˈæftɚwɚdz ɛkspɹˈɛs ɐ wˈɪʃ tə mˈɜːdɚ ðə ɹᵻkˈoːɹdɚ fɔːɹ hˌævɪŋ kˈɛpt ðˌɛm sˌoʊ lˈɔŋ ɪn səspˈɛns .|0
+LJ008-0294.wav|nˌɪɹli ɪndˈɛfɪnətli dᵻfˈɜːd .|0
+LJ047-0148.wav|ˌɔn ɑːktˈoʊbɚ twˈɛnti fˈaɪv ,|0
+LJ008-0111.wav|ðeɪ ˈɛntɚd ɐ dˈɑːlɚ stˈoʊŋ kˈoʊld ɹˈuːm , dˈɑːlɚɹ ænd wɜː pɹˈɛzəntli dʒˈɔɪnd baɪ ðə pɹˈɪzənɚ .|0
+LJ034-0042.wav|ðæt hiː kʊd ˈoʊnli tˈɛstᵻfˌaɪ wɪð sˈɜːtənti ðætðə pɹˈɪnt wʌz lˈɛs ðɐn θɹˈiː dˈeɪz ˈoʊld .|0
+LJ037-0234.wav|mˈɪsɪz . mˈɛɹi bɹˈɑːk , ðə wˈaɪf əvə mɪkˈænɪk hˌuː wˈɜːkt æt ðə stˈeɪʃən , wʌz ðɛɹ æt ðə tˈaɪm ænd ʃiː sˈɔː ɐ wˈaɪt mˈeɪl ,|0
+LJ040-0002.wav|tʃˈæptɚ sˈɛvən . lˈiː hˈɑːɹvi ˈɑːswəld : bˈækɡɹaʊnd ænd pˈɑːsᵻbəl mˈoʊɾɪvz , pˈɑːɹt wˌʌn .|0
+LJ045-0140.wav|ðɪ ˈɑːɹɡjuːmənts hiː jˈuːzd tə dʒˈʌstᵻfˌaɪ hɪz jˈuːs ʌvðɪ ˈeɪliəs sədʒˈɛst ðæt ˈɑːswəld mˌeɪhɐv kˈʌm tə θˈɪŋk ðætðə hˈoʊl wˈɜːld wʌz bᵻkˈʌmɪŋ ɪnvˈɑːlvd|0
+LJ012-0035.wav|ðə nˈʌmbɚ ænd nˈeɪmz ˌɔn wˈɑːtʃᵻz , wɜː kˈɛɹfəli ɹᵻmˈuːvd ɔːɹ əblˈɪɾɚɹˌeɪɾᵻd ˈæftɚ ðə ɡˈʊdz pˈæst ˌaʊɾəv hɪz hˈændz .|0
+LJ012-0250.wav|ɔnðə sˈɛvənθ dʒuːlˈaɪ , ˈeɪtiːn θˈɜːɾi sˈɛvən ,|0
+LJ016-0179.wav|kəntɹˈæktᵻd wɪð ʃˈɛɹɪfs ænd kənvˈiːnɚz tə wˈɜːk baɪ ðə dʒˈɑːb .|0
+LJ016-0138.wav|æɾə dˈɪstəns fɹʌmðə pɹˈɪzən .|0
+LJ027-0052.wav|ðiːz pɹˈɪnsɪpəlz ʌv həmˈɑːlədʒi ɑːɹ ᵻsˈɛnʃəl tʊ ɐ kɚɹˈɛkt ɪntˌɜːpɹɪtˈeɪʃən ʌvðə fˈækts ʌv mɔːɹfˈɑːlədʒi .|0
+LJ031-0134.wav|ˌɔn wˈʌn əkˈeɪʒən mˈɪsɪz . dʒˈɑːnsən , ɐkˈʌmpənid baɪ tˈuː sˈiːkɹᵻt sˈɜːvɪs ˈeɪdʒənts , lˈɛft ðə ɹˈuːm tə sˈiː mˈɪsɪz . kˈɛnədi ænd mˈɪsɪz . kˈɑːnæli .|0
+LJ019-0273.wav|wˌɪtʃ sˌɜː dʒˈɑːʃjuːə dʒˈɛb tˈoʊld ðə kəmˈɪɾi hiː kənsˈɪdɚd ðə pɹˈɑːpɚɹ ˈɛlɪmənts ʌv pˈiːnəl dˈɪsɪplˌɪn .|0
+LJ014-0110.wav|æt ðə fˈɜːst ðə bˈɑːksᵻz wɜːɹ ɪmpˈaʊndᵻd , ˈoʊpənd , ænd fˈaʊnd tə kəntˈeɪn mˈɛnɪəv oʊkˈɑːnɚz ɪfˈɛkts .|0
+LJ034-0160.wav|ˌɔn bɹˈɛnənz sˈʌbsᵻkwənt sˈɜːʔn̩ aɪdˈɛntɪfɪkˈeɪʃən ʌv lˈiː hˈɑːɹvi ˈɑːswəld æz ðə mˈæn hiː sˈɔː fˈaɪɚ ðə ɹˈaɪfəl .|0
+LJ038-0199.wav|ᵻlˈɛvən . ɪf aɪɐm ɐlˈaɪv ænd tˈeɪkən pɹˈɪzənɚ ,|0
+LJ014-0010.wav|jˈɛt hiː kʊd nˌɑːt ˌoʊvɚkˈʌm ðə stɹˈeɪndʒ fˌæsᵻnˈeɪʃən ɪt hˈæd fɔːɹ hˌɪm , ænd ɹᵻmˈeɪnd baɪ ðə sˈaɪd ʌvðə kˈɔːɹps tˈɪl ðə stɹˈɛtʃɚ kˈeɪm .|0
+LJ033-0047.wav|aɪ nˈoʊɾɪst wɛn aɪ wɛnt ˈaʊt ðætðə lˈaɪt wʌz ˈɔn , ˈɛnd kwˈoʊt ,|0
+LJ040-0027.wav|hiː wʌz nˈɛvɚ sˈæɾɪsfˌaɪd wɪð ˈɛnɪθˌɪŋ .|0
+LJ048-0228.wav|ænd ˈʌðɚz hˌuː wɜː pɹˈɛzənt sˈeɪ ðæt nˈoʊ ˈeɪdʒənt wʌz ɪnˈiːbɹɪˌeɪɾᵻd ɔːɹ ˈæktᵻd ɪmpɹˈɑːpɚli .|0
+LJ003-0111.wav|hiː wʌz ɪŋ kˈɑːnsɪkwəns pˌʊt ˌaʊɾəv ðə pɹətˈɛkʃən ʌv ðɛɹ ɪntˈɜːnəl lˈɔː , ˈɛnd kwˈoʊt . ðɛɹ kˈoʊd wʌzɐ sˈʌbdʒɛkt ʌv sˌʌm kjˌʊɹɹɪˈɔsɪɾi .|0
+LJ008-0258.wav|lˈɛt mˌiː ɹᵻtɹˈeɪs maɪ stˈɛps , ænd spˈiːk mˈoːɹ ɪn diːtˈeɪl ʌvðə tɹˈiːtmənt ʌvðə kəndˈɛmd ɪn ðoʊz blˈʌdθɜːsti ænd bɹˈuːɾəli ɪndˈɪfɹənt dˈeɪz ,|0
+LJ029-0022.wav|ðɪ ɚɹˈɪdʒɪnəl plˈæŋ kˈɔːld fɚðə pɹˈɛzɪdənt tə spˈɛnd ˈoʊnli wˈʌn dˈeɪ ɪnðə stˈeɪt , mˌeɪkɪŋ wˈɜːlwɪnd vˈɪzɪts tə dˈæləs , fˈɔːɹt wˈɜːθ , sˌæn æntˈoʊnɪˌoʊ , ænd hjˈuːstən .|0
+LJ004-0045.wav|mˈɪstɚ . stˈɜːdʒᵻz bˈoːɹn , sˌɜː dʒˈeɪmz mˈækɪntˌɑːʃ , sˌɜː dʒˈeɪmz skˈɑːɹlɪt , ænd wˈɪljəm wˈɪlbɚfˌoːɹs .|0

Demo/.ipynb_checkpoints/Inference_LJSpeech-checkpoint.ipynb

@@ -0,0 +1,554 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "9adb7bd1",
+   "metadata": {},
+   "source": [
+    "# StyleTTS 2 Demo (LJSpeech)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6108384d",
+   "metadata": {},
+   "source": [
+    "### Utils"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "96e173bf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "torch.manual_seed(0)\n",
+    "torch.backends.cudnn.benchmark = False\n",
+    "torch.backends.cudnn.deterministic = True\n",
+    "\n",
+    "import random\n",
+    "random.seed(0)\n",
+    "\n",
+    "import numpy as np\n",
+    "np.random.seed(0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "da84c60f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%cd .."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5a3ddcc8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# load packages\n",
+    "import time\n",
+    "import random\n",
+    "import yaml\n",
+    "from munch import Munch\n",
+    "import numpy as np\n",
+    "import torch\n",
+    "from torch import nn\n",
+    "import torch.nn.functional as F\n",
+    "import torchaudio\n",
+    "import librosa\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "\n",
+    "from models import *\n",
+    "from utils import *\n",
+    "from text_utils import TextCleaner\n",
+    "textclenaer = TextCleaner()\n",
+    "\n",
+    "%matplotlib inline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bbdc04c0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "device = 'cuda' if torch.cuda.is_available() else 'cpu'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "00ee05e1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "to_mel = torchaudio.transforms.MelSpectrogram(\n",
+    "    n_mels=80, n_fft=2048, win_length=1200, hop_length=300)\n",
+    "mean, std = -4, 4\n",
+    "\n",
+    "def length_to_mask(lengths):\n",
+    "    mask = torch.arange(lengths.max()).unsqueeze(0).expand(lengths.shape[0], -1).type_as(lengths)\n",
+    "    mask = torch.gt(mask+1, lengths.unsqueeze(1))\n",
+    "    return mask\n",
+    "\n",
+    "def preprocess(wave):\n",
+    "    wave_tensor = torch.from_numpy(wave).float()\n",
+    "    mel_tensor = to_mel(wave_tensor)\n",
+    "    mel_tensor = (torch.log(1e-5 + mel_tensor.unsqueeze(0)) - mean) / std\n",
+    "    return mel_tensor\n",
+    "\n",
+    "def compute_style(ref_dicts):\n",
+    "    reference_embeddings = {}\n",
+    "    for key, path in ref_dicts.items():\n",
+    "        wave, sr = librosa.load(path, sr=24000)\n",
+    "        audio, index = librosa.effects.trim(wave, top_db=30)\n",
+    "        if sr != 24000:\n",
+    "            audio = librosa.resample(audio, sr, 24000)\n",
+    "        mel_tensor = preprocess(audio).to(device)\n",
+    "\n",
+    "        with torch.no_grad():\n",
+    "            ref = model.style_encoder(mel_tensor.unsqueeze(1))\n",
+    "        reference_embeddings[key] = (ref.squeeze(1), audio)\n",
+    "    \n",
+    "    return reference_embeddings"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7b9cecbe",
+   "metadata": {},
+   "source": [
+    "### Load models"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "64fc4c0f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# load phonemizer\n",
+    "import phonemizer\n",
+    "global_phonemizer = phonemizer.backend.EspeakBackend(language='en-us', preserve_punctuation=True,  with_stress=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "48e7b644",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "config = yaml.safe_load(open(\"Models/LJSpeech/config.yml\"))\n",
+    "\n",
+    "# load pretrained ASR model\n",
+    "ASR_config = config.get('ASR_config', False)\n",
+    "ASR_path = config.get('ASR_path', False)\n",
+    "text_aligner = load_ASR_models(ASR_path, ASR_config)\n",
+    "\n",
+    "# load pretrained F0 model\n",
+    "F0_path = config.get('F0_path', False)\n",
+    "pitch_extractor = load_F0_models(F0_path)\n",
+    "\n",
+    "# load BERT model\n",
+    "from Utils.PLBERT.util import load_plbert\n",
+    "BERT_path = config.get('PLBERT_dir', False)\n",
+    "plbert = load_plbert(BERT_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ffc18cf7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = build_model(recursive_munch(config['model_params']), text_aligner, pitch_extractor, plbert)\n",
+    "_ = [model[key].eval() for key in model]\n",
+    "_ = [model[key].to(device) for key in model]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "64529d5c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "params_whole = torch.load(\"Models/LJSpeech/epoch_2nd_00100.pth\", map_location='cpu')\n",
+    "params = params_whole['net']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "895d9706",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for key in model:\n",
+    "    if key in params:\n",
+    "        print('%s loaded' % key)\n",
+    "        try:\n",
+    "            model[key].load_state_dict(params[key])\n",
+    "        except:\n",
+    "            from collections import OrderedDict\n",
+    "            state_dict = params[key]\n",
+    "            new_state_dict = OrderedDict()\n",
+    "            for k, v in state_dict.items():\n",
+    "                name = k[7:] # remove `module.`\n",
+    "                new_state_dict[name] = v\n",
+    "            # load params\n",
+    "            model[key].load_state_dict(new_state_dict, strict=False)\n",
+    "#             except:\n",
+    "#                 _load(params[key], model[key])\n",
+    "_ = [model[key].eval() for key in model]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c1a59db2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from Modules.diffusion.sampler import DiffusionSampler, ADPM2Sampler, KarrasSchedule"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e30985ab",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sampler = DiffusionSampler(\n",
+    "    model.diffusion.diffusion,\n",
+    "    sampler=ADPM2Sampler(),\n",
+    "    sigma_schedule=KarrasSchedule(sigma_min=0.0001, sigma_max=3.0, rho=9.0), # empirical parameters\n",
+    "    clamp=False\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b803110e",
+   "metadata": {},
+   "source": [
+    "### Synthesize speech"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "24655f46",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# synthesize a text\n",
+    "text = ''' StyleTTS 2 is a text-to-speech model that leverages style diffusion and adversarial training with large speech language models to achieve human-level text-to-speech synthesis. '''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ca57469c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def inference(text, noise, diffusion_steps=5, embedding_scale=1):\n",
+    "    text = text.strip()\n",
+    "    text = text.replace('\"', '')\n",
+    "    ps = global_phonemizer.phonemize([text])\n",
+    "    ps = word_tokenize(ps[0])\n",
+    "    ps = ' '.join(ps)\n",
+    "\n",
+    "    tokens = textclenaer(ps)\n",
+    "    tokens.insert(0, 0)\n",
+    "    tokens = torch.LongTensor(tokens).to(device).unsqueeze(0)\n",
+    "    \n",
+    "    with torch.no_grad():\n",
+    "        input_lengths = torch.LongTensor([tokens.shape[-1]]).to(tokens.device)\n",
+    "        text_mask = length_to_mask(input_lengths).to(tokens.device)\n",
+    "\n",
+    "        t_en = model.text_encoder(tokens, input_lengths, text_mask)\n",
+    "        bert_dur = model.bert(tokens, attention_mask=(~text_mask).int())\n",
+    "        d_en = model.bert_encoder(bert_dur).transpose(-1, -2) \n",
+    "\n",
+    "        s_pred = sampler(noise, \n",
+    "              embedding=bert_dur[0].unsqueeze(0), num_steps=diffusion_steps,\n",
+    "              embedding_scale=embedding_scale).squeeze(0)\n",
+    "\n",
+    "        s = s_pred[:, 128:]\n",
+    "        ref = s_pred[:, :128]\n",
+    "\n",
+    "        d = model.predictor.text_encoder(d_en, s, input_lengths, text_mask)\n",
+    "\n",
+    "        x, _ = model.predictor.lstm(d)\n",
+    "        duration = model.predictor.duration_proj(x)\n",
+    "        duration = torch.sigmoid(duration).sum(axis=-1)\n",
+    "        pred_dur = torch.round(duration.squeeze()).clamp(min=1)\n",
+    "\n",
+    "        pred_dur[-1] += 5\n",
+    "\n",
+    "        pred_aln_trg = torch.zeros(input_lengths, int(pred_dur.sum().data))\n",
+    "        c_frame = 0\n",
+    "        for i in range(pred_aln_trg.size(0)):\n",
+    "            pred_aln_trg[i, c_frame:c_frame + int(pred_dur[i].data)] = 1\n",
+    "            c_frame += int(pred_dur[i].data)\n",
+    "\n",
+    "        # encode prosody\n",
+    "        en = (d.transpose(-1, -2) @ pred_aln_trg.unsqueeze(0).to(device))\n",
+    "        F0_pred, N_pred = model.predictor.F0Ntrain(en, s)\n",
+    "        out = model.decoder((t_en @ pred_aln_trg.unsqueeze(0).to(device)), \n",
+    "                                F0_pred, N_pred, ref.squeeze().unsqueeze(0))\n",
+    "        \n",
+    "    return out.squeeze().cpu().numpy()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d438ef4f",
+   "metadata": {},
+   "source": [
+    "#### Basic synthesis (5 diffusion steps)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d3d7f7d5",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "start = time.time()\n",
+    "noise = torch.randn(1,1,256).to(device)\n",
+    "wav = inference(text, noise, diffusion_steps=5, embedding_scale=1)\n",
+    "rtf = (time.time() - start) / (len(wav) / 24000)\n",
+    "print(f\"RTF = {rtf:5f}\")\n",
+    "import IPython.display as ipd\n",
+    "display(ipd.Audio(wav, rate=24000))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2d5d9df0",
+   "metadata": {},
+   "source": [
+    "#### With higher diffusion steps (more diverse)\n",
+    "Since the sampler is ancestral, the higher the stpes, the more diverse the samples are, with the cost of slower synthesis speed."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a10129fd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "start = time.time()\n",
+    "noise = torch.randn(1,1,256).to(device)\n",
+    "wav = inference(text, noise, diffusion_steps=10, embedding_scale=1)\n",
+    "rtf = (time.time() - start) / (len(wav) / 24000)\n",
+    "print(f\"RTF = {rtf:5f}\")\n",
+    "import IPython.display as ipd\n",
+    "display(ipd.Audio(wav, rate=24000))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1877ea15",
+   "metadata": {},
+   "source": [
+    "### Speech expressiveness\n",
+    "The following section recreates the samples shown in [Section 6](https://styletts2.github.io/#emo) of the demo page."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4c4777b7",
+   "metadata": {},
+   "source": [
+    "#### With embedding_scale=1\n",
+    "This is the classifier-free guidance scale. The higher the scale, the more conditional the style is to the input text and hence more emotional. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c29ea2f0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "texts = {}\n",
+    "texts['Happy'] = \"We are happy to invite you to join us on a journey to the past, where we will visit the most amazing monuments ever built by human hands.\"\n",
+    "texts['Sad'] = \"I am sorry to say that we have suffered a severe setback in our efforts to restore prosperity and confidence.\"\n",
+    "texts['Angry'] = \"The field of astronomy is a joke! Its theories are based on flawed observations and biased interpretations!\"\n",
+    "texts['Surprised'] = \"I can't believe it! You mean to tell me that you have discovered a new species of bacteria in this pond?\"\n",
+    "\n",
+    "for k,v in texts.items():\n",
+    "    noise = torch.randn(1,1,256).to(device)\n",
+    "    wav = inference(v, noise, diffusion_steps=10, embedding_scale=1)\n",
+    "    print(k + \": \")\n",
+    "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3c89499f",
+   "metadata": {},
+   "source": [
+    "#### With embedding_scale=2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f73be3aa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "texts = {}\n",
+    "texts['Happy'] = \"We are happy to invite you to join us on a journey to the past, where we will visit the most amazing monuments ever built by human hands.\"\n",
+    "texts['Sad'] = \"I am sorry to say that we have suffered a severe setback in our efforts to restore prosperity and confidence.\"\n",
+    "texts['Angry'] = \"The field of astronomy is a joke! Its theories are based on flawed observations and biased interpretations!\"\n",
+    "texts['Surprised'] = \"I can't believe it! You mean to tell me that you have discovered a new species of bacteria in this pond?\"\n",
+    "\n",
+    "for k,v in texts.items():\n",
+    "    noise = torch.randn(1,1,256).to(device)\n",
+    "    wav = inference(v, noise, diffusion_steps=10, embedding_scale=2) # embedding_scale=2 for more pronounced emotion\n",
+    "    print(k + \": \")\n",
+    "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9320da63",
+   "metadata": {},
+   "source": [
+    "### Long-form generation\n",
+    "This section includes basic implementation of Algorithm 1 in the paper for consistent longform audio generation. The example passage is taken from [Section 5](https://styletts2.github.io/#long) of the demo page. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cdd4db51",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "passage = '''If the supply of fruit is greater than the family needs, it may be made a source of income by sending the fresh fruit to the market if there is one near enough, or by preserving, canning, and making jelly for sale. To make such an enterprise a success the fruit and work must be first class. There is magic in the word \"Homemade,\" when the product appeals to the eye and the palate; but many careless and incompetent people have found to their sorrow that this word has not magic enough to float inferior goods on the market. As a rule large canning and preserving establishments are clean and have the best appliances, and they employ chemists and skilled labor. The home product must be very good to compete with the attractive goods that are sent out from such establishments. Yet for first-class homemade products there is a market in all large cities. All first-class grocers have customers who purchase such goods.'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ebb941c8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def LFinference(text, s_prev, noise, alpha=0.7, diffusion_steps=5, embedding_scale=1):\n",
+    "    text = text.strip()\n",
+    "    text = text.replace('\"', '')\n",
+    "    ps = global_phonemizer.phonemize([text])\n",
+    "    ps = word_tokenize(ps[0])\n",
+    "    ps = ' '.join(ps)\n",
+    "\n",
+    "    tokens = textclenaer(ps)\n",
+    "    tokens.insert(0, 0)\n",
+    "    tokens = torch.LongTensor(tokens).to(device).unsqueeze(0)\n",
+    "    \n",
+    "    with torch.no_grad():\n",
+    "        input_lengths = torch.LongTensor([tokens.shape[-1]]).to(tokens.device)\n",
+    "        text_mask = length_to_mask(input_lengths).to(tokens.device)\n",
+    "\n",
+    "        t_en = model.text_encoder(tokens, input_lengths, text_mask)\n",
+    "        bert_dur = model.bert(tokens, attention_mask=(~text_mask).int())\n",
+    "        d_en = model.bert_encoder(bert_dur).transpose(-1, -2) \n",
+    "\n",
+    "        s_pred = sampler(noise, \n",
+    "              embedding=bert_dur[0].unsqueeze(0), num_steps=diffusion_steps,\n",
+    "              embedding_scale=embedding_scale).squeeze(0)\n",
+    "        \n",
+    "        if s_prev is not None:\n",
+    "            # convex combination of previous and current style\n",
+    "            s_pred = alpha * s_prev + (1 - alpha) * s_pred\n",
+    "        \n",
+    "        s = s_pred[:, 128:]\n",
+    "        ref = s_pred[:, :128]\n",
+    "\n",
+    "        d = model.predictor.text_encoder(d_en, s, input_lengths, text_mask)\n",
+    "\n",
+    "        x, _ = model.predictor.lstm(d)\n",
+    "        duration = model.predictor.duration_proj(x)\n",
+    "        duration = torch.sigmoid(duration).sum(axis=-1)\n",
+    "        pred_dur = torch.round(duration.squeeze()).clamp(min=1)\n",
+    "\n",
+    "        pred_aln_trg = torch.zeros(input_lengths, int(pred_dur.sum().data))\n",
+    "        c_frame = 0\n",
+    "        for i in range(pred_aln_trg.size(0)):\n",
+    "            pred_aln_trg[i, c_frame:c_frame + int(pred_dur[i].data)] = 1\n",
+    "            c_frame += int(pred_dur[i].data)\n",
+    "\n",
+    "        # encode prosody\n",
+    "        en = (d.transpose(-1, -2) @ pred_aln_trg.unsqueeze(0).to(device))\n",
+    "        F0_pred, N_pred = model.predictor.F0Ntrain(en, s)\n",
+    "        out = model.decoder((t_en @ pred_aln_trg.unsqueeze(0).to(device)), \n",
+    "                                F0_pred, N_pred, ref.squeeze().unsqueeze(0))\n",
+    "        \n",
+    "    return out.squeeze().cpu().numpy(), s_pred"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7ca0ef2e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sentences = passage.split('.') # simple split by comma\n",
+    "wavs = []\n",
+    "s_prev = None\n",
+    "for text in sentences:\n",
+    "    if text.strip() == \"\": continue\n",
+    "    text += '.' # add it back\n",
+    "    noise = torch.randn(1,1,256).to(device)\n",
+    "    wav, s_prev = LFinference(text, s_prev, noise, alpha=0.7, diffusion_steps=10, embedding_scale=1.5)\n",
+    "    wavs.append(wav)\n",
+    "display(ipd.Audio(np.concatenate(wavs), rate=24000, normalize=False))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "NLP",
+   "language": "python",
+   "name": "nlp"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

Demo/.ipynb_checkpoints/Inference_LibriTTS-checkpoint.ipynb

@@ -0,0 +1,1155 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "9adb7bd1",
+   "metadata": {},
+   "source": [
+    "# StyleTTS 2 Demo (LibriTTS)\n",
+    "\n",
+    "Before you run the following cells, please make sure you have downloaded [reference_audio.zip](https://huggingface.co/yl4579/StyleTTS2-LibriTTS/resolve/main/reference_audio.zip) and unzipped it under the `demo` folder."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6108384d",
+   "metadata": {},
+   "source": [
+    "### Utils"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "96e173bf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "torch.manual_seed(0)\n",
+    "torch.backends.cudnn.benchmark = False\n",
+    "torch.backends.cudnn.deterministic = True\n",
+    "\n",
+    "import random\n",
+    "random.seed(0)\n",
+    "\n",
+    "import numpy as np\n",
+    "np.random.seed(0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "da84c60f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%cd .."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5a3ddcc8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# load packages\n",
+    "import time\n",
+    "import random\n",
+    "import yaml\n",
+    "from munch import Munch\n",
+    "import numpy as np\n",
+    "import torch\n",
+    "from torch import nn\n",
+    "import torch.nn.functional as F\n",
+    "import torchaudio\n",
+    "import librosa\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "\n",
+    "from models import *\n",
+    "from utils import *\n",
+    "from text_utils import TextCleaner\n",
+    "textclenaer = TextCleaner()\n",
+    "\n",
+    "%matplotlib inline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "00ee05e1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "to_mel = torchaudio.transforms.MelSpectrogram(\n",
+    "    n_mels=80, n_fft=2048, win_length=1200, hop_length=300)\n",
+    "mean, std = -4, 4\n",
+    "\n",
+    "def length_to_mask(lengths):\n",
+    "    mask = torch.arange(lengths.max()).unsqueeze(0).expand(lengths.shape[0], -1).type_as(lengths)\n",
+    "    mask = torch.gt(mask+1, lengths.unsqueeze(1))\n",
+    "    return mask\n",
+    "\n",
+    "def preprocess(wave):\n",
+    "    wave_tensor = torch.from_numpy(wave).float()\n",
+    "    mel_tensor = to_mel(wave_tensor)\n",
+    "    mel_tensor = (torch.log(1e-5 + mel_tensor.unsqueeze(0)) - mean) / std\n",
+    "    return mel_tensor\n",
+    "\n",
+    "def compute_style(path):\n",
+    "    wave, sr = librosa.load(path, sr=24000)\n",
+    "    audio, index = librosa.effects.trim(wave, top_db=30)\n",
+    "    if sr != 24000:\n",
+    "        audio = librosa.resample(audio, sr, 24000)\n",
+    "    mel_tensor = preprocess(audio).to(device)\n",
+    "\n",
+    "    with torch.no_grad():\n",
+    "        ref_s = model.style_encoder(mel_tensor.unsqueeze(1))\n",
+    "        ref_p = model.predictor_encoder(mel_tensor.unsqueeze(1))\n",
+    "\n",
+    "    return torch.cat([ref_s, ref_p], dim=1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bbdc04c0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "device = 'cuda' if torch.cuda.is_available() else 'cpu'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7b9cecbe",
+   "metadata": {},
+   "source": [
+    "### Load models"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "64fc4c0f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# load phonemizer\n",
+    "import phonemizer\n",
+    "global_phonemizer = phonemizer.backend.EspeakBackend(language='en-us', preserve_punctuation=True,  with_stress=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "48e7b644",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "config = yaml.safe_load(open(\"Models/LibriTTS/config.yml\"))\n",
+    "\n",
+    "# load pretrained ASR model\n",
+    "ASR_config = config.get('ASR_config', False)\n",
+    "ASR_path = config.get('ASR_path', False)\n",
+    "text_aligner = load_ASR_models(ASR_path, ASR_config)\n",
+    "\n",
+    "# load pretrained F0 model\n",
+    "F0_path = config.get('F0_path', False)\n",
+    "pitch_extractor = load_F0_models(F0_path)\n",
+    "\n",
+    "# load BERT model\n",
+    "from Utils.PLBERT.util import load_plbert\n",
+    "BERT_path = config.get('PLBERT_dir', False)\n",
+    "plbert = load_plbert(BERT_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ffc18cf7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model_params = recursive_munch(config['model_params'])\n",
+    "model = build_model(model_params, text_aligner, pitch_extractor, plbert)\n",
+    "_ = [model[key].eval() for key in model]\n",
+    "_ = [model[key].to(device) for key in model]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "64529d5c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "params_whole = torch.load(\"Models/LibriTTS/epochs_2nd_00020.pth\", map_location='cpu')\n",
+    "params = params_whole['net']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "895d9706",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for key in model:\n",
+    "    if key in params:\n",
+    "        print('%s loaded' % key)\n",
+    "        try:\n",
+    "            model[key].load_state_dict(params[key])\n",
+    "        except:\n",
+    "            from collections import OrderedDict\n",
+    "            state_dict = params[key]\n",
+    "            new_state_dict = OrderedDict()\n",
+    "            for k, v in state_dict.items():\n",
+    "                name = k[7:] # remove `module.`\n",
+    "                new_state_dict[name] = v\n",
+    "            # load params\n",
+    "            model[key].load_state_dict(new_state_dict, strict=False)\n",
+    "#             except:\n",
+    "#                 _load(params[key], model[key])\n",
+    "_ = [model[key].eval() for key in model]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c1a59db2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from Modules.diffusion.sampler import DiffusionSampler, ADPM2Sampler, KarrasSchedule"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e30985ab",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sampler = DiffusionSampler(\n",
+    "    model.diffusion.diffusion,\n",
+    "    sampler=ADPM2Sampler(),\n",
+    "    sigma_schedule=KarrasSchedule(sigma_min=0.0001, sigma_max=3.0, rho=9.0), # empirical parameters\n",
+    "    clamp=False\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b803110e",
+   "metadata": {},
+   "source": [
+    "### Synthesize speech"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ca57469c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def inference(text, ref_s, alpha = 0.3, beta = 0.7, diffusion_steps=5, embedding_scale=1):\n",
+    "    text = text.strip()\n",
+    "    ps = global_phonemizer.phonemize([text])\n",
+    "    ps = word_tokenize(ps[0])\n",
+    "    ps = ' '.join(ps)\n",
+    "    tokens = textclenaer(ps)\n",
+    "    tokens.insert(0, 0)\n",
+    "    tokens = torch.LongTensor(tokens).to(device).unsqueeze(0)\n",
+    "    \n",
+    "    with torch.no_grad():\n",
+    "        input_lengths = torch.LongTensor([tokens.shape[-1]]).to(device)\n",
+    "        text_mask = length_to_mask(input_lengths).to(device)\n",
+    "\n",
+    "        t_en = model.text_encoder(tokens, input_lengths, text_mask)\n",
+    "        bert_dur = model.bert(tokens, attention_mask=(~text_mask).int())\n",
+    "        d_en = model.bert_encoder(bert_dur).transpose(-1, -2) \n",
+    "\n",
+    "        s_pred = sampler(noise = torch.randn((1, 256)).unsqueeze(1).to(device), \n",
+    "                                          embedding=bert_dur,\n",
+    "                                          embedding_scale=embedding_scale,\n",
+    "                                            features=ref_s, # reference from the same speaker as the embedding\n",
+    "                                             num_steps=diffusion_steps).squeeze(1)\n",
+    "\n",
+    "\n",
+    "        s = s_pred[:, 128:]\n",
+    "        ref = s_pred[:, :128]\n",
+    "\n",
+    "        ref = alpha * ref + (1 - alpha)  * ref_s[:, :128]\n",
+    "        s = beta * s + (1 - beta)  * ref_s[:, 128:]\n",
+    "\n",
+    "        d = model.predictor.text_encoder(d_en, \n",
+    "                                         s, input_lengths, text_mask)\n",
+    "\n",
+    "        x, _ = model.predictor.lstm(d)\n",
+    "        duration = model.predictor.duration_proj(x)\n",
+    "\n",
+    "        duration = torch.sigmoid(duration).sum(axis=-1)\n",
+    "        pred_dur = torch.round(duration.squeeze()).clamp(min=1)\n",
+    "\n",
+    "\n",
+    "        pred_aln_trg = torch.zeros(input_lengths, int(pred_dur.sum().data))\n",
+    "        c_frame = 0\n",
+    "        for i in range(pred_aln_trg.size(0)):\n",
+    "            pred_aln_trg[i, c_frame:c_frame + int(pred_dur[i].data)] = 1\n",
+    "            c_frame += int(pred_dur[i].data)\n",
+    "\n",
+    "        # encode prosody\n",
+    "        en = (d.transpose(-1, -2) @ pred_aln_trg.unsqueeze(0).to(device))\n",
+    "        if model_params.decoder.type == \"hifigan\":\n",
+    "            asr_new = torch.zeros_like(en)\n",
+    "            asr_new[:, :, 0] = en[:, :, 0]\n",
+    "            asr_new[:, :, 1:] = en[:, :, 0:-1]\n",
+    "            en = asr_new\n",
+    "\n",
+    "        F0_pred, N_pred = model.predictor.F0Ntrain(en, s)\n",
+    "\n",
+    "        asr = (t_en @ pred_aln_trg.unsqueeze(0).to(device))\n",
+    "        if model_params.decoder.type == \"hifigan\":\n",
+    "            asr_new = torch.zeros_like(asr)\n",
+    "            asr_new[:, :, 0] = asr[:, :, 0]\n",
+    "            asr_new[:, :, 1:] = asr[:, :, 0:-1]\n",
+    "            asr = asr_new\n",
+    "\n",
+    "        out = model.decoder(asr, \n",
+    "                                F0_pred, N_pred, ref.squeeze().unsqueeze(0))\n",
+    "    \n",
+    "        \n",
+    "    return out.squeeze().cpu().numpy()[..., :-50] # weird pulse at the end of the model, need to be fixed later"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d438ef4f",
+   "metadata": {},
+   "source": [
+    "#### Basic synthesis (5 diffusion steps, seen speakers)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cace9787",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "text = ''' StyleTTS 2 is a text to speech model that leverages style diffusion and adversarial training with large speech language models to achieve human level text to speech synthesis. '''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7c88f461",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "reference_dicts = {}\n",
+    "reference_dicts['696_92939'] = \"Demo/reference_audio/696_92939_000016_000006.wav\"\n",
+    "reference_dicts['1789_142896'] = \"Demo/reference_audio/1789_142896_000022_000005.wav\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "16e8ac60",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "start = time.time()\n",
+    "noise = torch.randn(1,1,256).to(device)\n",
+    "for k, path in reference_dicts.items():\n",
+    "    ref_s = compute_style(path)\n",
+    "    \n",
+    "    wav = inference(text, ref_s, alpha=0.3, beta=0.7, diffusion_steps=5, embedding_scale=1)\n",
+    "    rtf = (time.time() - start) / (len(wav) / 24000)\n",
+    "    print(f\"RTF = {rtf:5f}\")\n",
+    "    import IPython.display as ipd\n",
+    "    print(k + ' Synthesized:')\n",
+    "    display(ipd.Audio(wav, rate=24000, normalize=False))\n",
+    "    print('Reference:')\n",
+    "    display(ipd.Audio(path, rate=24000, normalize=False))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "14838708",
+   "metadata": {},
+   "source": [
+    "#### With higher diffusion steps (more diverse)\n",
+    "\n",
+    "Since the sampler is ancestral, the higher the stpes, the more diverse the samples are, with the cost of slower synthesis speed."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6fbff03b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "noise = torch.randn(1,1,256).to(device)\n",
+    "for k, path in reference_dicts.items():\n",
+    "    ref_s = compute_style(path)\n",
+    "    start = time.time()\n",
+    "    wav = inference(text, ref_s, alpha=0.3, beta=0.7, diffusion_steps=10, embedding_scale=1)\n",
+    "    rtf = (time.time() - start) / (len(wav) / 24000)\n",
+    "    print(f\"RTF = {rtf:5f}\")\n",
+    "    import IPython.display as ipd\n",
+    "    print(k + ' Synthesized:')\n",
+    "    display(ipd.Audio(wav, rate=24000, normalize=False))\n",
+    "    print(k + ' Reference:')\n",
+    "    display(ipd.Audio(path, rate=24000, normalize=False))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7e6867fd",
+   "metadata": {},
+   "source": [
+    "#### Basic synthesis (5 diffusion steps, umseen speakers)\n",
+    "The following samples are to reproduce samples in [Section 4](https://styletts2.github.io/#libri) of the demo page. All spsakers are unseen during training. You can compare the generated samples to popular zero-shot TTS models like Vall-E and NaturalSpeech 2."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f4e8faa0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "reference_dicts = {}\n",
+    "# format: (path, text)\n",
+    "reference_dicts['1221-135767'] = (\"Demo/reference_audio/1221-135767-0014.wav\", \"Yea, his honourable worship is within, but he hath a godly minister or two with him, and likewise a leech.\")\n",
+    "reference_dicts['5639-40744'] = (\"Demo/reference_audio/5639-40744-0020.wav\", \"Thus did this humane and right minded father comfort his unhappy daughter, and her mother embracing her again, did all she could to soothe her feelings.\")\n",
+    "reference_dicts['908-157963'] = (\"Demo/reference_audio/908-157963-0027.wav\", \"And lay me down in my cold bed and leave my shining lot.\")\n",
+    "reference_dicts['4077-13754'] = (\"Demo/reference_audio/4077-13754-0000.wav\", \"The army found the people in poverty and left them in comparative wealth.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "653f1406",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "noise = torch.randn(1,1,256).to(device)\n",
+    "for k, v in reference_dicts.items():\n",
+    "    path, text = v\n",
+    "    ref_s = compute_style(path)\n",
+    "    start = time.time()\n",
+    "    wav = inference(text, ref_s, alpha=0.3, beta=0.7, diffusion_steps=5, embedding_scale=1)\n",
+    "    rtf = (time.time() - start) / (len(wav) / 24000)\n",
+    "    print(f\"RTF = {rtf:5f}\")\n",
+    "    import IPython.display as ipd\n",
+    "    print(k + ' Synthesized: ' + text)\n",
+    "    display(ipd.Audio(wav, rate=24000, normalize=False))\n",
+    "    print(k + ' Reference:')\n",
+    "    display(ipd.Audio(path, rate=24000, normalize=False))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "141e91b3",
+   "metadata": {},
+   "source": [
+    "### Speech expressiveness\n",
+    "\n",
+    "The following section recreates the samples shown in [Section 6](https://styletts2.github.io/#emo) of the demo page. The speaker reference used is `1221-135767-0014.wav`, which is unseen during training. \n",
+    "\n",
+    "#### With `embedding_scale=1`\n",
+    "This is the classifier-free guidance scale. The higher the scale, the more conditional the style is to the input text and hence more emotional.\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "81addda4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ref_s = compute_style(\"Demo/reference_audio/1221-135767-0014.wav\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "be1b2a11",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "texts = {}\n",
+    "texts['Happy'] = \"We are happy to invite you to join us on a journey to the past, where we will visit the most amazing monuments ever built by human hands.\"\n",
+    "texts['Sad'] = \"I am sorry to say that we have suffered a severe setback in our efforts to restore prosperity and confidence.\"\n",
+    "texts['Angry'] = \"The field of astronomy is a joke! Its theories are based on flawed observations and biased interpretations!\"\n",
+    "texts['Surprised'] = \"I can't believe it! You mean to tell me that you have discovered a new species of bacteria in this pond?\"\n",
+    "\n",
+    "for k,v in texts.items():\n",
+    "    wav = inference(v, ref_s, diffusion_steps=10, alpha=0.3, beta=0.7, embedding_scale=1)\n",
+    "    print(k + \": \")\n",
+    "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "96d262b8",
+   "metadata": {},
+   "source": [
+    "#### With `embedding_scale=2`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3e7d40b4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "texts = {}\n",
+    "texts['Happy'] = \"We are happy to invite you to join us on a journey to the past, where we will visit the most amazing monuments ever built by human hands.\"\n",
+    "texts['Sad'] = \"I am sorry to say that we have suffered a severe setback in our efforts to restore prosperity and confidence.\"\n",
+    "texts['Angry'] = \"The field of astronomy is a joke! Its theories are based on flawed observations and biased interpretations!\"\n",
+    "texts['Surprised'] = \"I can't believe it! You mean to tell me that you have discovered a new species of bacteria in this pond?\"\n",
+    "\n",
+    "for k,v in texts.items():\n",
+    "    noise = torch.randn(1,1,256).to(device)\n",
+    "    wav = inference(v, ref_s, diffusion_steps=10, alpha=0.3, beta=0.7, embedding_scale=2)\n",
+    "    print(k + \": \")\n",
+    "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "402b2bd6",
+   "metadata": {},
+   "source": [
+    "#### With `embedding_scale=2, alpha = 0.5, beta = 0.9`\n",
+    "`alpha` and `beta` is the factor to determine much we use the style sampled based on the text instead of the reference. The higher the value of `alpha` and `beta`, the more suitable the style it is to the text but less similar to the reference. Using higher beta makes the synthesized speech more emotional, at the cost of lower similarity to the reference. `alpha` determines the timbre of the speaker while `beta` determines the prosody. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "599de5d5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "texts = {}\n",
+    "texts['Happy'] = \"We are happy to invite you to join us on a journey to the past, where we will visit the most amazing monuments ever built by human hands.\"\n",
+    "texts['Sad'] = \"I am sorry to say that we have suffered a severe setback in our efforts to restore prosperity and confidence.\"\n",
+    "texts['Angry'] = \"The field of astronomy is a joke! Its theories are based on flawed observations and biased interpretations!\"\n",
+    "texts['Surprised'] = \"I can't believe it! You mean to tell me that you have discovered a new species of bacteria in this pond?\"\n",
+    "\n",
+    "for k,v in texts.items():\n",
+    "    noise = torch.randn(1,1,256).to(device)\n",
+    "    wav = inference(v, ref_s, diffusion_steps=10, alpha=0.5, beta=0.9, embedding_scale=2)\n",
+    "    print(k + \": \")\n",
+    "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "48548866",
+   "metadata": {},
+   "source": [
+    "### Zero-shot speaker adaptation\n",
+    "This section recreates the \"Acoustic Environment Maintenance\" and \"Speaker’s Emotion Maintenance\" demo in [Section 4](https://styletts2.github.io/#libri) of the demo page. You can compare the generated samples to popular zero-shot TTS models like Vall-E. Note that the model was trained only on LibriTTS, which is about 250 times fewer data compared to those used to trian Vall-E with similar or better effect for these maintainance. "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "23e81572",
+   "metadata": {},
+   "source": [
+    "#### Acoustic Environment Maintenance\n",
+    "\n",
+    "Since we want to maintain the acoustic environment in the speaker (timbre), we set  `alpha = 0` to make the speaker as closer to the reference as possible while only changing the prosody according to the text.  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8087bccb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "reference_dicts = {}\n",
+    "# format: (path, text)\n",
+    "reference_dicts['3'] = (\"Demo/reference_audio/3.wav\", \"As friends thing I definitely I've got more male friends.\")\n",
+    "reference_dicts['4'] = (\"Demo/reference_audio/4.wav\", \"Everything is run by computer but you got to know how to think before you can do a computer.\")\n",
+    "reference_dicts['5'] = (\"Demo/reference_audio/5.wav\", \"Then out in LA you guys got a whole another ball game within California to worry about.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1e99c200",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "noise = torch.randn(1,1,256).to(device)\n",
+    "for k, v in reference_dicts.items():\n",
+    "    path, text = v\n",
+    "    ref_s = compute_style(path)\n",
+    "    start = time.time()\n",
+    "    wav = inference(text, ref_s, alpha=0.0, beta=0.5, diffusion_steps=5, embedding_scale=1)\n",
+    "    rtf = (time.time() - start) / (len(wav) / 24000)\n",
+    "    print(f\"RTF = {rtf:5f}\")\n",
+    "    import IPython.display as ipd\n",
+    "    print('Synthesized: ' + text)\n",
+    "    display(ipd.Audio(wav, rate=24000, normalize=False))\n",
+    "    print('Reference:')\n",
+    "    display(ipd.Audio(path, rate=24000, normalize=False))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7d56505d",
+   "metadata": {},
+   "source": [
+    "#### Speaker’s Emotion Maintenance\n",
+    "\n",
+    "Since we want to maintain the emotion in the speaker (prosody), we set  `beta = 0.1` to make the speaker as closer to the reference as possible while having some diversity thruogh the slight timbre change."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f90179e7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "reference_dicts = {}\n",
+    "# format: (path, text)\n",
+    "reference_dicts['Anger'] = (\"Demo/reference_audio/anger.wav\", \"We have to reduce the number of plastic bags.\")\n",
+    "reference_dicts['Sleepy'] = (\"Demo/reference_audio/sleepy.wav\", \"We have to reduce the number of plastic bags.\")\n",
+    "reference_dicts['Amused'] = (\"Demo/reference_audio/amused.wav\", \"We have to reduce the number of plastic bags.\")\n",
+    "reference_dicts['Disgusted'] = (\"Demo/reference_audio/disgusted.wav\", \"We have to reduce the number of plastic bags.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2e6bdfed",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "noise = torch.randn(1,1,256).to(device)\n",
+    "for k, v in reference_dicts.items():\n",
+    "    path, text = v\n",
+    "    ref_s = compute_style(path)\n",
+    "    start = time.time()\n",
+    "    wav = inference(text, ref_s, alpha=0.3, beta=0.1, diffusion_steps=10, embedding_scale=1)\n",
+    "    rtf = (time.time() - start) / (len(wav) / 24000)\n",
+    "    print(f\"RTF = {rtf:5f}\")\n",
+    "    import IPython.display as ipd\n",
+    "    print(k + ' Synthesized: ' + text)\n",
+    "    display(ipd.Audio(wav, rate=24000, normalize=False))\n",
+    "    print(k + ' Reference:')\n",
+    "    display(ipd.Audio(path, rate=24000, normalize=False))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "37ae3963",
+   "metadata": {},
+   "source": [
+    "### Longform Narration\n",
+    "\n",
+    "This section includes basic implementation of Algorithm 1 in the paper for consistent longform audio generation. The example passage is taken from [Section 5](https://styletts2.github.io/#long) of the demo page."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f12a716b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "passage = '''If the supply of fruit is greater than the family needs, it may be made a source of income by sending the fresh fruit to the market if there is one near enough, or by preserving, canning, and making jelly for sale. To make such an enterprise a success the fruit and work must be first class. There is magic in the word \"Homemade,\" when the product appeals to the eye and the palate; but many careless and incompetent people have found to their sorrow that this word has not magic enough to float inferior goods on the market. As a rule large canning and preserving establishments are clean and have the best appliances, and they employ chemists and skilled labor. The home product must be very good to compete with the attractive goods that are sent out from such establishments. Yet for first class home made products there is a market in all large cities. All first-class grocers have customers who purchase such goods.'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a1a38079",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def LFinference(text, s_prev, ref_s, alpha = 0.3, beta = 0.7, t = 0.7, diffusion_steps=5, embedding_scale=1):\n",
+    "    text = text.strip()\n",
+    "    ps = global_phonemizer.phonemize([text])\n",
+    "    ps = word_tokenize(ps[0])\n",
+    "    ps = ' '.join(ps)\n",
+    "    ps = ps.replace('``', '\"')\n",
+    "    ps = ps.replace(\"''\", '\"')\n",
+    "\n",
+    "    tokens = textclenaer(ps)\n",
+    "    tokens.insert(0, 0)\n",
+    "    tokens = torch.LongTensor(tokens).to(device).unsqueeze(0)\n",
+    "    \n",
+    "    with torch.no_grad():\n",
+    "        input_lengths = torch.LongTensor([tokens.shape[-1]]).to(device)\n",
+    "        text_mask = length_to_mask(input_lengths).to(device)\n",
+    "\n",
+    "        t_en = model.text_encoder(tokens, input_lengths, text_mask)\n",
+    "        bert_dur = model.bert(tokens, attention_mask=(~text_mask).int())\n",
+    "        d_en = model.bert_encoder(bert_dur).transpose(-1, -2) \n",
+    "\n",
+    "        s_pred = sampler(noise = torch.randn((1, 256)).unsqueeze(1).to(device), \n",
+    "                                          embedding=bert_dur,\n",
+    "                                          embedding_scale=embedding_scale,\n",
+    "                                            features=ref_s, # reference from the same speaker as the embedding\n",
+    "                                             num_steps=diffusion_steps).squeeze(1)\n",
+    "        \n",
+    "        if s_prev is not None:\n",
+    "            # convex combination of previous and current style\n",
+    "            s_pred = t * s_prev + (1 - t) * s_pred\n",
+    "        \n",
+    "        s = s_pred[:, 128:]\n",
+    "        ref = s_pred[:, :128]\n",
+    "        \n",
+    "        ref = alpha * ref + (1 - alpha)  * ref_s[:, :128]\n",
+    "        s = beta * s + (1 - beta)  * ref_s[:, 128:]\n",
+    "\n",
+    "        s_pred = torch.cat([ref, s], dim=-1)\n",
+    "\n",
+    "        d = model.predictor.text_encoder(d_en, \n",
+    "                                         s, input_lengths, text_mask)\n",
+    "\n",
+    "        x, _ = model.predictor.lstm(d)\n",
+    "        duration = model.predictor.duration_proj(x)\n",
+    "\n",
+    "        duration = torch.sigmoid(duration).sum(axis=-1)\n",
+    "        pred_dur = torch.round(duration.squeeze()).clamp(min=1)\n",
+    "\n",
+    "\n",
+    "        pred_aln_trg = torch.zeros(input_lengths, int(pred_dur.sum().data))\n",
+    "        c_frame = 0\n",
+    "        for i in range(pred_aln_trg.size(0)):\n",
+    "            pred_aln_trg[i, c_frame:c_frame + int(pred_dur[i].data)] = 1\n",
+    "            c_frame += int(pred_dur[i].data)\n",
+    "\n",
+    "        # encode prosody\n",
+    "        en = (d.transpose(-1, -2) @ pred_aln_trg.unsqueeze(0).to(device))\n",
+    "        if model_params.decoder.type == \"hifigan\":\n",
+    "            asr_new = torch.zeros_like(en)\n",
+    "            asr_new[:, :, 0] = en[:, :, 0]\n",
+    "            asr_new[:, :, 1:] = en[:, :, 0:-1]\n",
+    "            en = asr_new\n",
+    "\n",
+    "        F0_pred, N_pred = model.predictor.F0Ntrain(en, s)\n",
+    "\n",
+    "        asr = (t_en @ pred_aln_trg.unsqueeze(0).to(device))\n",
+    "        if model_params.decoder.type == \"hifigan\":\n",
+    "            asr_new = torch.zeros_like(asr)\n",
+    "            asr_new[:, :, 0] = asr[:, :, 0]\n",
+    "            asr_new[:, :, 1:] = asr[:, :, 0:-1]\n",
+    "            asr = asr_new\n",
+    "\n",
+    "        out = model.decoder(asr, \n",
+    "                                F0_pred, N_pred, ref.squeeze().unsqueeze(0))\n",
+    "    \n",
+    "        \n",
+    "    return out.squeeze().cpu().numpy()[..., :-100], s_pred # weird pulse at the end of the model, need to be fixed later"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e9088f7a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# unseen speaker\n",
+    "path = \"Demo/reference_audio/1221-135767-0014.wav\"\n",
+    "s_ref = compute_style(path)\n",
+    "sentences = passage.split('.') # simple split by comma\n",
+    "wavs = []\n",
+    "s_prev = None\n",
+    "for text in sentences:\n",
+    "    if text.strip() == \"\": continue\n",
+    "    text += '.' # add it back\n",
+    "    \n",
+    "    wav, s_prev = LFinference(text, \n",
+    "                              s_prev, \n",
+    "                              s_ref, \n",
+    "                              alpha = 0.3, \n",
+    "                              beta = 0.9,  # make it more suitable for the text\n",
+    "                              t = 0.7, \n",
+    "                              diffusion_steps=10, embedding_scale=1.5)\n",
+    "    wavs.append(wav)\n",
+    "print('Synthesized: ')\n",
+    "display(ipd.Audio(np.concatenate(wavs), rate=24000, normalize=False))\n",
+    "print('Reference: ')\n",
+    "display(ipd.Audio(path, rate=24000, normalize=False))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7517b657",
+   "metadata": {},
+   "source": [
+    "### Style Transfer\n",
+    "\n",
+    "The following section demostrates the style transfer capacity for unseen speakers in [Section 6](https://styletts2.github.io/#emo) of the demo page. For this, we set `alpha=0.5, beta = 0.9` for the most pronounced effects (mostly using the sampled style). "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ed95d0f7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def STinference(text, ref_s, ref_text, alpha = 0.3, beta = 0.7, diffusion_steps=5, embedding_scale=1):\n",
+    "    text = text.strip()\n",
+    "    ps = global_phonemizer.phonemize([text])\n",
+    "    ps = word_tokenize(ps[0])\n",
+    "    ps = ' '.join(ps)\n",
+    "\n",
+    "    tokens = textclenaer(ps)\n",
+    "    tokens.insert(0, 0)\n",
+    "    tokens = torch.LongTensor(tokens).to(device).unsqueeze(0)\n",
+    "    \n",
+    "    ref_text = ref_text.strip()\n",
+    "    ps = global_phonemizer.phonemize([ref_text])\n",
+    "    ps = word_tokenize(ps[0])\n",
+    "    ps = ' '.join(ps)\n",
+    "\n",
+    "    ref_tokens = textclenaer(ps)\n",
+    "    ref_tokens.insert(0, 0)\n",
+    "    ref_tokens = torch.LongTensor(ref_tokens).to(device).unsqueeze(0)\n",
+    "    \n",
+    "    \n",
+    "    with torch.no_grad():\n",
+    "        input_lengths = torch.LongTensor([tokens.shape[-1]]).to(device)\n",
+    "        text_mask = length_to_mask(input_lengths).to(device)\n",
+    "\n",
+    "        t_en = model.text_encoder(tokens, input_lengths, text_mask)\n",
+    "        bert_dur = model.bert(tokens, attention_mask=(~text_mask).int())\n",
+    "        d_en = model.bert_encoder(bert_dur).transpose(-1, -2) \n",
+    "        \n",
+    "        ref_input_lengths = torch.LongTensor([ref_tokens.shape[-1]]).to(device)\n",
+    "        ref_text_mask = length_to_mask(ref_input_lengths).to(device)\n",
+    "        ref_bert_dur = model.bert(ref_tokens, attention_mask=(~ref_text_mask).int())\n",
+    "        s_pred = sampler(noise = torch.randn((1, 256)).unsqueeze(1).to(device), \n",
+    "                                          embedding=bert_dur,\n",
+    "                                          embedding_scale=embedding_scale,\n",
+    "                                            features=ref_s, # reference from the same speaker as the embedding\n",
+    "                                             num_steps=diffusion_steps).squeeze(1)\n",
+    "\n",
+    "\n",
+    "        s = s_pred[:, 128:]\n",
+    "        ref = s_pred[:, :128]\n",
+    "\n",
+    "        ref = alpha * ref + (1 - alpha)  * ref_s[:, :128]\n",
+    "        s = beta * s + (1 - beta)  * ref_s[:, 128:]\n",
+    "\n",
+    "        d = model.predictor.text_encoder(d_en, \n",
+    "                                         s, input_lengths, text_mask)\n",
+    "\n",
+    "        x, _ = model.predictor.lstm(d)\n",
+    "        duration = model.predictor.duration_proj(x)\n",
+    "\n",
+    "        duration = torch.sigmoid(duration).sum(axis=-1)\n",
+    "        pred_dur = torch.round(duration.squeeze()).clamp(min=1)\n",
+    "\n",
+    "\n",
+    "        pred_aln_trg = torch.zeros(input_lengths, int(pred_dur.sum().data))\n",
+    "        c_frame = 0\n",
+    "        for i in range(pred_aln_trg.size(0)):\n",
+    "            pred_aln_trg[i, c_frame:c_frame + int(pred_dur[i].data)] = 1\n",
+    "            c_frame += int(pred_dur[i].data)\n",
+    "\n",
+    "        # encode prosody\n",
+    "        en = (d.transpose(-1, -2) @ pred_aln_trg.unsqueeze(0).to(device))\n",
+    "        if model_params.decoder.type == \"hifigan\":\n",
+    "            asr_new = torch.zeros_like(en)\n",
+    "            asr_new[:, :, 0] = en[:, :, 0]\n",
+    "            asr_new[:, :, 1:] = en[:, :, 0:-1]\n",
+    "            en = asr_new\n",
+    "\n",
+    "        F0_pred, N_pred = model.predictor.F0Ntrain(en, s)\n",
+    "\n",
+    "        asr = (t_en @ pred_aln_trg.unsqueeze(0).to(device))\n",
+    "        if model_params.decoder.type == \"hifigan\":\n",
+    "            asr_new = torch.zeros_like(asr)\n",
+    "            asr_new[:, :, 0] = asr[:, :, 0]\n",
+    "            asr_new[:, :, 1:] = asr[:, :, 0:-1]\n",
+    "            asr = asr_new\n",
+    "\n",
+    "        out = model.decoder(asr, \n",
+    "                                F0_pred, N_pred, ref.squeeze().unsqueeze(0))\n",
+    "    \n",
+    "        \n",
+    "    return out.squeeze().cpu().numpy()[..., :-50] # weird pulse at the end of the model, need to be fixed later"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ec3f0da4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# reference texts to sample styles\n",
+    "\n",
+    "ref_texts = {}\n",
+    "ref_texts['Happy'] = \"We are happy to invite you to join us on a journey to the past, where we will visit the most amazing monuments ever built by human hands.\"\n",
+    "ref_texts['Sad'] = \"I am sorry to say that we have suffered a severe setback in our efforts to restore prosperity and confidence.\"\n",
+    "ref_texts['Angry'] = \"The field of astronomy is a joke! Its theories are based on flawed observations and biased interpretations!\"\n",
+    "ref_texts['Surprised'] = \"I can't believe it! You mean to tell me that you have discovered a new species of bacteria in this pond?\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6d0a3825",
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "path = \"Demo/reference_audio/1221-135767-0014.wav\"\n",
+    "s_ref = compute_style(path)\n",
+    "\n",
+    "text = \"Yea, his honourable worship is within, but he hath a godly minister or two with him, and likewise a leech.\"\n",
+    "for k,v in texts.items():\n",
+    "    wav = STinference(text, s_ref, v, diffusion_steps=10, alpha=0.5, beta=0.9, embedding_scale=1.5)\n",
+    "    print(k + \": \")\n",
+    "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6750aed9",
+   "metadata": {},
+   "source": [
+    "### Speech diversity\n",
+    "\n",
+    "This section reproduces samples in [Section 7](https://styletts2.github.io/#var) of the demo page. \n",
+    "\n",
+    "`alpha` and `beta` determine the diversity of the synthesized speech. There are two extreme cases:\n",
+    "- If `alpha = 1` and `beta = 1`, the synthesized speech sounds the most dissimilar to the reference speaker, but it is also the most diverse (each time you synthesize a speech it will be totally different). \n",
+    "- If `alpha = 0` and `beta = 0`, the synthesized speech sounds the most siimlar to the reference speaker, but it is deterministic (i.e., the sampled style is not used for speech synthesis). \n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f6ae0aa5",
+   "metadata": {},
+   "source": [
+    "#### Default setting (`alpha = 0.3, beta=0.7`)\n",
+    "This setting uses 70% of the reference timbre and 30% of the reference prosody and use the diffusion model to sample them based on the text. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "36dc0148",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# unseen speaker\n",
+    "path = \"Demo/reference_audio/1221-135767-0014.wav\"\n",
+    "ref_s = compute_style(path)\n",
+    "\n",
+    "text = \"How much variation is there?\"\n",
+    "for _ in range(5):\n",
+    "    wav = inference(text, ref_s, diffusion_steps=10, alpha=0.3, beta=0.7, embedding_scale=1)\n",
+    "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "bf9ef421",
+   "metadata": {},
+   "source": [
+    "#### Less diverse setting (`alpha = 0.1, beta=0.3`)\n",
+    "This setting uses 90% of the reference timbre and 70% of the reference prosody. This makes it more similar to the reference speaker at cost of less diverse samples. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9ba406bd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# unseen speaker\n",
+    "path = \"Demo/reference_audio/1221-135767-0014.wav\"\n",
+    "ref_s = compute_style(path)\n",
+    "\n",
+    "text = \"How much variation is there?\"\n",
+    "for _ in range(5):\n",
+    "    wav = inference(text, ref_s, diffusion_steps=10, alpha=0.1, beta=0.3, embedding_scale=1)\n",
+    "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a38fe464",
+   "metadata": {},
+   "source": [
+    "#### More diverse setting (`alpha = 0.5, beta=0.95`)\n",
+    "This setting uses 50% of the reference timbre and 5% of the reference prosody (so it uses 100% of the sampled prosody, which makes it more diverse), but this makes it more dissimilar to the reference speaker.  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5f25bf94",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# unseen speaker\n",
+    "path = \"Demo/reference_audio/1221-135767-0014.wav\"\n",
+    "ref_s = compute_style(path)\n",
+    "\n",
+    "text = \"How much variation is there?\"\n",
+    "for _ in range(5):\n",
+    "    wav = inference(text, ref_s, diffusion_steps=10, alpha=0.5, beta=0.95, embedding_scale=1)\n",
+    "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "21c3a071",
+   "metadata": {},
+   "source": [
+    "#### Extreme setting (`alpha = 1, beta=1`)\n",
+    "This setting uses 0% of the reference timbre and prosody and use the diffusion model to sample the entire style. This makes the speaker very dissimilar to the reference speaker. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fff8bab1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# unseen speaker\n",
+    "path = \"Demo/reference_audio/1221-135767-0014.wav\"\n",
+    "ref_s = compute_style(path)\n",
+    "\n",
+    "text = \"How much variation is there?\"\n",
+    "for _ in range(5):\n",
+    "    wav = inference(text, ref_s, diffusion_steps=10, alpha=1, beta=1, embedding_scale=1)\n",
+    "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a8741e5a",
+   "metadata": {},
+   "source": [
+    "#### No variation (`alpha = 0, beta=0`)\n",
+    "This setting uses 0% of the reference timbre and prosody and use the diffusion model to sample the entire style. This makes the speaker very similar to the reference speaker, but there is no variation. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e55dd281",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# unseen speaker\n",
+    "path = \"Demo/reference_audio/1221-135767-0014.wav\"\n",
+    "ref_s = compute_style(path)\n",
+    "\n",
+    "text = \"How much variation is there?\"\n",
+    "for _ in range(5):\n",
+    "    wav = inference(text, ref_s, diffusion_steps=10, alpha=0, beta=0, embedding_scale=1)\n",
+    "    display(ipd.Audio(wav, rate=24000, normalize=False))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d5e86423",
+   "metadata": {},
+   "source": [
+    "### Extra fun!\n",
+    "\n",
+    "Here we clone some of the authors' voice of the StyleTTS 2 papers with a few seconds of the recording in the wild. None of the voices is in the dataset and all authors agreed to have their voices cloned here."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6f558314",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "text = ''' StyleTTS 2 is a text to speech model that leverages style diffusion and adversarial training with large speech language models to achieve human level text to speech synthesis. '''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "caa5747c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "reference_dicts = {}\n",
+    "reference_dicts['Yinghao'] = \"Demo/reference_audio/Yinghao.wav\"\n",
+    "reference_dicts['Gavin'] = \"Demo/reference_audio/Gavin.wav\"\n",
+    "reference_dicts['Vinay'] = \"Demo/reference_audio/Vinay.wav\"\n",
+    "reference_dicts['Nima'] = \"Demo/reference_audio/Nima.wav\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "44a4cea1",
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "start = time.time()\n",
+    "noise = torch.randn(1,1,256).to(device)\n",
+    "for k, path in reference_dicts.items():\n",
+    "    ref_s = compute_style(path)\n",
+    "    \n",
+    "    wav = inference(text, ref_s, alpha=0.1, beta=0.5, diffusion_steps=5, embedding_scale=1)\n",
+    "    rtf = (time.time() - start) / (len(wav) / 24000)\n",
+    "    print('Speaker: ' + k)\n",
+    "    import IPython.display as ipd\n",
+    "    print('Synthesized:')\n",
+    "    display(ipd.Audio(wav, rate=24000, normalize=False))\n",
+    "    print('Reference:')\n",
+    "    display(ipd.Audio(path, rate=24000, normalize=False))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "NLP",
+   "language": "python",
+   "name": "nlp"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

Demo/Inference_LJSpeech.ipynb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce816595cf645bece49b912627fa5f317d26f72f429f1a67566f0d2149a63213
+size 10573186

Demo/Inference_LibriTTS.ipynb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65da24dc5388d4af6e65b698794d7daa1a21d15b329905b32a72b0e5070a336c
+size 36102818

Demo/reference_audio/James.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ace7afc8b1a55978c22e10f862374465e654f70ba72cea894e32751fdb4be6ec
+size 4663340

Demo/reference_audio/James1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3cd888091538093ba760e9f8ceaa9f4d88bd861ab8c4dea5db7e79e6bc893d0
+size 1419308

Demo/reference_audio/James2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4fca55c26fea4b130566738a8632b935592f2aa29042f5d4e4ba0180eddcf96
+size 1757284

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Aaron (Yinghao) Li
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Models/LJSpeech/config.yml

@@ -0,0 +1,22 @@
+{ASR_config: Utils/ASR/config.yml, ASR_path: Utils/ASR/epoch_00080.pth, F0_path: Utils/JDC/bst.t7,
+  PLBERT_dir: Utils/PLBERT/, batch_size: 16, data_params: {OOD_data: Data/OOD_texts.txt,
+    min_length: 50, root_path: /local/LJSpeech-1.1/wavs, train_data: Data/train_list.txt,
+    val_data: Data/val_list.txt}, device: cuda, epochs_1st: 200, epochs_2nd: 100,
+  first_stage_path: first_stage.pth, load_only_params: false, log_dir: Models/LJSpeech,
+  log_interval: 10, loss_params: {TMA_epoch: 50, diff_epoch: 20, joint_epoch: 50,
+    lambda_F0: 1.0, lambda_ce: 20.0, lambda_diff: 1.0, lambda_dur: 1.0, lambda_gen: 1.0,
+    lambda_mel: 5.0, lambda_mono: 1.0, lambda_norm: 1.0, lambda_s2s: 1.0, lambda_slm: 1.0,
+    lambda_sty: 1.0}, max_len: 400, model_params: {decoder: {gen_istft_hop_size: 5,
+      gen_istft_n_fft: 20, resblock_dilation_sizes: [[1, 3, 5], [1, 3, 5], [1, 3,
+          5]], resblock_kernel_sizes: [3, 7, 11], type: istftnet, upsample_initial_channel: 512,
+      upsample_kernel_sizes: [20, 12], upsample_rates: [10, 6]}, diffusion: {dist: {
+        estimate_sigma_data: true, mean: -3.0, sigma_data: 0.45731624995853165, std: 1.0},
+      embedding_mask_proba: 0.1, transformer: {head_features: 64, multiplier: 2, num_heads: 8,
+        num_layers: 3}}, dim_in: 64, dropout: 0.2, hidden_dim: 512, max_conv_dim: 512,
+    max_dur: 50, multispeaker: false, n_layer: 3, n_mels: 80, n_token: 178, slm: {
+      hidden: 768, initial_channel: 64, model: microsoft/wavlm-base-plus, nlayers: 13,
+      sr: 16000}, style_dim: 128}, optimizer_params: {bert_lr: 1.0e-05, ft_lr: 1.0e-05,
+    lr: 0.0001}, preprocess_params: {spect_params: {hop_length: 300, n_fft: 2048,
+      win_length: 1200}, sr: 24000}, pretrained_model:,
+  save_freq: 2, second_stage_load_pretrained: true, slmadv_params: {batch_percentage: 0.5,
+    iter: 10, max_len: 500, min_len: 400, scale: 0.01, sig: 1.5, thresh: 5}}

Models/LJSpeech/epoch_2nd_00100.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fead81b4ccee69f296efc5e4ead0973a51cb645ac24195c995977d4a9ee8d780
+size 749716474

Models/LibriTTS/config.yml

@@ -0,0 +1,21 @@
+{ASR_config: Utils/ASR/config.yml, ASR_path: Utils/ASR/epoch_00080.pth, F0_path: Utils/JDC/bst.t7,
+  PLBERT_dir: Utils/PLBERT/, batch_size: 8, data_params: {OOD_data: Data/OOD_texts.txt,
+    min_length: 50, root_path: '', train_data: Data/train_list.txt, val_data: Data/val_list.txt},
+  device: cuda, epochs_1st: 40, epochs_2nd: 25, first_stage_path: first_stage.pth,
+  load_only_params: false, log_dir: Models/LibriTTS, log_interval: 10, loss_params: {
+    TMA_epoch: 4, diff_epoch: 0, joint_epoch: 0, lambda_F0: 1.0, lambda_ce: 20.0,
+    lambda_diff: 1.0, lambda_dur: 1.0, lambda_gen: 1.0, lambda_mel: 5.0, lambda_mono: 1.0,
+    lambda_norm: 1.0, lambda_s2s: 1.0, lambda_slm: 1.0, lambda_sty: 1.0}, max_len: 300,
+  model_params: {decoder: {resblock_dilation_sizes: [[1, 3, 5], [1, 3, 5], [1, 3,
+          5]], resblock_kernel_sizes: [3, 7, 11], type: hifigan, upsample_initial_channel: 512,
+      upsample_kernel_sizes: [20, 10, 6, 4], upsample_rates: [10, 5, 3, 2]}, diffusion: {
+      dist: {estimate_sigma_data: true, mean: -3.0, sigma_data: 0.19926648961191362,
+        std: 1.0}, embedding_mask_proba: 0.1, transformer: {head_features: 64, multiplier: 2,
+        num_heads: 8, num_layers: 3}}, dim_in: 64, dropout: 0.2, hidden_dim: 512,
+    max_conv_dim: 512, max_dur: 50, multispeaker: true, n_layer: 3, n_mels: 80, n_token: 178,
+    slm: {hidden: 768, initial_channel: 64, model: microsoft/wavlm-base-plus, nlayers: 13,
+      sr: 16000}, style_dim: 128}, optimizer_params: {bert_lr: 1.0e-05, ft_lr: 1.0e-05,
+    lr: 0.0001}, preprocess_params: {spect_params: {hop_length: 300, n_fft: 2048,
+      win_length: 1200}, sr: 24000}, pretrained_model: Models/LibriTTS/epoch_2nd_00002.pth,
+  save_freq: 1, second_stage_load_pretrained: true, slmadv_params: {batch_percentage: 0.5,
+    iter: 20, max_len: 500, min_len: 400, scale: 0.01, sig: 1.5, thresh: 5}}

Models/LibriTTS/epochs_2nd_00020.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1164ffe19a17449d2c722234cecaf2836b35a698fb8ffd42562d2663657dca0a
+size 771390526

Modules/__init__.py

@@ -0,0 +1 @@
+

Modules/diffusion/__init__.py

@@ -0,0 +1 @@
+