Upload 6 files

datasets/.gitignore

@@ -0,0 +1,3 @@
+LJSpeech-1.1/
+MBSpeech-1.0/
+*.tar.gz

datasets/data_loader.py

@@ -0,0 +1,121 @@
+import random
+
+import numpy as np
+import torch
+from torch.utils.data.dataloader import default_collate, DataLoader
+from torch.utils.data.sampler import Sampler
+
+__all__ = ['Text2MelDataLoader', 'SSRNDataLoader']
+
+
+class Text2MelDataLoader(DataLoader):
+    def __init__(self, text2mel_dataset, batch_size, mode='train', num_workers=8):
+        if mode == 'train':
+            text2mel_dataset.slice(0, -batch_size)
+        elif mode == 'valid':
+            text2mel_dataset.slice(len(text2mel_dataset) - batch_size, -1)
+        else:
+            raise ValueError("mode must be either 'train' or 'valid'")
+        super().__init__(text2mel_dataset,
+                         batch_size=batch_size,
+                         num_workers=num_workers,
+                         collate_fn=collate_fn,
+                         shuffle=True)
+
+
+class SSRNDataLoader(DataLoader):
+    def __init__(self, ssrn_dataset, batch_size, mode='train', num_workers=8):
+        if mode == 'train':
+            ssrn_dataset.slice(0, -batch_size)
+            super().__init__(ssrn_dataset,
+                             batch_size=batch_size,
+                             num_workers=num_workers,
+                             collate_fn=collate_fn,
+                             sampler=PartiallyRandomizedSimilarTimeLengthSampler(lengths=ssrn_dataset.text_lengths,
+                                                                                 data_source=None,
+                                                                                 batch_size=batch_size))
+        elif mode == 'valid':
+            ssrn_dataset.slice(len(ssrn_dataset) - batch_size, -1)
+            super().__init__(ssrn_dataset,
+                             batch_size=batch_size,
+                             num_workers=num_workers,
+                             collate_fn=collate_fn,
+                             shuffle=True)
+        else:
+            raise ValueError("mode must be either 'train' or 'valid'")
+
+
+def collate_fn(batch):
+    keys = batch[0].keys()
+    max_lengths = {key: 0 for key in keys}
+    collated_batch = {key: [] for key in keys}
+
+    # find out the max lengths
+    for row in batch:
+        for key in keys:
+            max_lengths[key] = max(max_lengths[key], row[key].shape[0])
+
+    # pad to the max lengths
+    for row in batch:
+        for key in keys:
+            array = row[key]
+            dim = len(array.shape)
+            assert dim == 1 or dim == 2
+            # TODO: because of pre processing, later we want to have (n_mels, T)
+            if dim == 1:
+                padded_array = np.pad(array, (0, max_lengths[key] - array.shape[0]), mode='constant')
+            else:
+                padded_array = np.pad(array, ((0, max_lengths[key] - array.shape[0]), (0, 0)), mode='constant')
+            collated_batch[key].append(padded_array)
+
+    # use the default_collate to convert to tensors
+    for key in keys:
+        collated_batch[key] = default_collate(collated_batch[key])
+    return collated_batch
+
+
+class PartiallyRandomizedSimilarTimeLengthSampler(Sampler):
+    """Copied from: https://github.com/r9y9/deepvoice3_pytorch/blob/master/train.py.
+    Partially randomized sampler
+    1. Sort by lengths
+    2. Pick a small patch and randomize it
+    3. Permutate mini-batches
+    """
+
+    def __init__(self, lengths, data_source, batch_size=16, batch_group_size=None, permutate=True):
+        super().__init__(data_source)
+        self.lengths, self.sorted_indices = torch.sort(torch.LongTensor(lengths))
+        self.batch_size = batch_size
+        if batch_group_size is None:
+            batch_group_size = min(batch_size * 32, len(self.lengths))
+            if batch_group_size % batch_size != 0:
+                batch_group_size -= batch_group_size % batch_size
+
+        self.batch_group_size = batch_group_size
+        assert batch_group_size % batch_size == 0
+        self.permutate = permutate
+
+    def __iter__(self):
+        indices = self.sorted_indices.clone()
+        batch_group_size = self.batch_group_size
+        s, e = 0, 0
+        for i in range(len(indices) // batch_group_size):
+            s = i * batch_group_size
+            e = s + batch_group_size
+            random.shuffle(indices[s:e])
+
+        # Permutate batches
+        if self.permutate:
+            perm = np.arange(len(indices[:e]) // self.batch_size)
+            random.shuffle(perm)
+            indices[:e] = indices[:e].view(-1, self.batch_size)[perm, :].view(-1)
+
+        # Handle last elements
+        s += batch_group_size
+        if s < len(indices):
+            random.shuffle(indices[s:])
+
+        return iter(indices)
+
+    def __len__(self):
+        return len(self.sorted_indices)

datasets/emovdb.py

@@ -0,0 +1,82 @@
+"""Data loader for the Emovdb dataset. See: https://github.com/numediart/EmoV-DB"""
+import os
+import re
+import codecs
+import unicodedata
+import numpy as np
+from audio import preprocess
+
+from torch.utils.data import Dataset
+
+vocab = "PE abcdefghijklmnopqrstuvwxyz'.?"  # P: Padding, E: EOS.
+char2idx = {char: idx for idx, char in enumerate(vocab)}
+idx2char = {idx: char for idx, char in enumerate(vocab)}
+
+
+def text_normalize(text):
+    text = ''.join(char for char in unicodedata.normalize('NFD', text)
+                   if unicodedata.category(char) != 'Mn')  # Strip accents
+
+    text = text.lower()
+    text = re.sub("[^{}]".format(vocab), " ", text)
+    text = re.sub("[ ]+", " ", text)
+    return text
+
+
+def read_metadata(metadata_file):
+    fnames, text_lengths, texts = [], [], []
+    transcript = os.path.join(metadata_file)
+    lines = codecs.open(transcript, 'r', 'utf-8').readlines()
+    for line in lines:
+        fname, text = line.strip().split("|")
+
+        fnames.append(fname)
+
+        text = text_normalize(text) + "E"  # E: EOS
+        text = [char2idx[char] for char in text]
+        text_lengths.append(len(text))
+        texts.append(np.array(text, np.long))
+
+    return fnames, text_lengths, texts
+
+
+def get_test_data(sentences, max_n):
+    normalized_sentences = [text_normalize(line).strip() + "E" for line in sentences]  # text normalization, E: EOS
+    texts = np.zeros((len(normalized_sentences), max_n + 1), np.long)
+    for i, sent in enumerate(normalized_sentences):
+        texts[i, :len(sent)] = [char2idx[char] for char in sent]
+    return texts
+
+
+class Emovdb(Dataset):
+    def __init__(self, keys, dir_name='/home/brihi16142/work2/processed_emovdb_disgust'):
+        self.keys = keys
+        self.path = os.path.join(os.path.dirname(os.path.realpath(__file__)), dir_name)
+        self.fnames, self.text_lengths, self.texts = read_metadata(os.path.join(self.path, 'transcript_bea.csv'))
+        preprocess(dir_name, self)
+        print('Generated mels and mags')
+
+    def slice(self, start, end):
+        self.fnames = self.fnames[start:end]
+        self.text_lengths = self.text_lengths[start:end]
+        self.texts = self.texts[start:end]
+
+    def __len__(self):
+        return len(self.fnames)
+
+    def __getitem__(self, index):
+        data = {}
+        if 'texts' in self.keys:
+            data['texts'] = self.texts[index]
+        if 'mels' in self.keys:
+            # (39, 80)
+            data['mels'] = np.load(os.path.join(self.path, 'mels', "%s.npy" % self.fnames[index]))
+        if 'mags' in self.keys:
+            # (39, 80)
+            data['mags'] = np.load(os.path.join(self.path, 'mags', "%s.npy" % self.fnames[index]))
+        if 'mel_gates' in self.keys:
+            data['mel_gates'] = np.ones(data['mels'].shape[0], dtype=np.int)  # TODO: because pre processing!
+        if 'mag_gates' in self.keys:
+            data['mag_gates'] = np.ones(data['mags'].shape[0], dtype=np.int)  # TODO: because pre processing!
+        return data
+

datasets/lj_speech.py

@@ -0,0 +1,78 @@
+"""Data loader for the LJSpeech dataset. See: https://keithito.com/LJ-Speech-Dataset/"""
+import os
+import re
+import codecs
+import unicodedata
+import numpy as np
+
+from torch.utils.data import Dataset
+
+vocab = "PE abcdefghijklmnopqrstuvwxyz'.?"  # P: Padding, E: EOS.
+char2idx = {char: idx for idx, char in enumerate(vocab)}
+idx2char = {idx: char for idx, char in enumerate(vocab)}
+
+
+def text_normalize(text):
+    text = ''.join(char for char in unicodedata.normalize('NFD', text)
+                   if unicodedata.category(char) != 'Mn')  # Strip accents
+
+    text = text.lower()
+    text = re.sub("[^{}]".format(vocab), " ", text)
+    text = re.sub("[ ]+", " ", text)
+    return text
+
+
+def read_metadata(metadata_file):
+    fnames, text_lengths, texts = [], [], []
+    transcript = os.path.join(metadata_file)
+    lines = codecs.open(transcript, 'r', 'utf-8').readlines()
+    for line in lines:
+        fname, _, text = line.strip().split("|")
+
+        fnames.append(fname)
+
+        text = text_normalize(text) + "E"  # E: EOS
+        text = [char2idx[char] for char in text]
+        text_lengths.append(len(text))
+        texts.append(np.array(text, np.long))
+
+    return fnames, text_lengths, texts
+
+
+def get_test_data(sentences, max_n):
+    normalized_sentences = [text_normalize(line).strip() + "E" for line in sentences]  # text normalization, E: EOS
+    texts = np.zeros((len(normalized_sentences), max_n + 1), np.long)
+    for i, sent in enumerate(normalized_sentences):
+        texts[i, :len(sent)] = [char2idx[char] for char in sent]
+    return texts
+
+
+class LJSpeech(Dataset):
+    def __init__(self, keys, dir_name='LJSpeech-1.1'):
+        self.keys = keys
+        self.path = os.path.join(os.path.dirname(os.path.realpath(__file__)), dir_name)
+        self.fnames, self.text_lengths, self.texts = read_metadata(os.path.join(self.path, 'metadata.csv'))
+
+    def slice(self, start, end):
+        self.fnames = self.fnames[start:end]
+        self.text_lengths = self.text_lengths[start:end]
+        self.texts = self.texts[start:end]
+
+    def __len__(self):
+        return len(self.fnames)
+
+    def __getitem__(self, index):
+        data = {}
+        if 'texts' in self.keys:
+            data['texts'] = self.texts[index]
+        if 'mels' in self.keys:
+            # (39, 80)
+            data['mels'] = np.load(os.path.join(self.path, 'mels', "%s.npy" % self.fnames[index]))
+        if 'mags' in self.keys:
+            # (39, 80)
+            data['mags'] = np.load(os.path.join(self.path, 'mags', "%s.npy" % self.fnames[index]))
+        if 'mel_gates' in self.keys:
+            data['mel_gates'] = np.ones(data['mels'].shape[0], dtype=np.int)  # TODO: because pre processing!
+        if 'mag_gates' in self.keys:
+            data['mag_gates'] = np.ones(data['mags'].shape[0], dtype=np.int)  # TODO: because pre processing!
+        return data

datasets/mb_speech.py

@@ -0,0 +1,139 @@
+"""Data loader for the Mongolian Bible dataset."""
+import os
+import codecs
+import numpy as np
+
+from torch.utils.data import Dataset
+
+vocab = "PE абвгдеёжзийклмноөпрстуүфхцчшъыьэюя-.,!?"  # P: Padding, E: EOS.
+char2idx = {char: idx for idx, char in enumerate(vocab)}
+idx2char = {idx: char for idx, char in enumerate(vocab)}
+
+
+def text_normalize(text):
+    text = text.lower()
+    # text = text.replace(",", "'")
+    # text = text.replace("!", "?")
+    for c in "-—:":
+        text = text.replace(c, "-")
+    for c in "()\"«»“”'":
+        text = text.replace(c, ",")
+    return text
+
+
+def read_metadata(metadata_file):
+    fnames, text_lengths, texts = [], [], []
+    transcript = os.path.join(metadata_file)
+    lines = codecs.open(transcript, 'r', 'utf-8').readlines()
+    for line in lines:
+        fname, _, text = line.strip().split("|")
+
+        fnames.append(fname)
+
+        text = text_normalize(text) + "E"  # E: EOS
+        text = [char2idx[char] for char in text]
+        text_lengths.append(len(text))
+        texts.append(np.array(text, np.long))
+
+    return fnames, text_lengths, texts
+
+
+def get_test_data(sentences, max_n):
+    normalized_sentences = [text_normalize(line).strip() + "E" for line in sentences]  # text normalization, E: EOS
+    texts = np.zeros((len(normalized_sentences), max_n + 1), np.long)
+    for i, sent in enumerate(normalized_sentences):
+        texts[i, :len(sent)] = [char2idx[char] for char in sent]
+    return texts
+
+
+class MBSpeech(Dataset):
+    def __init__(self, keys, dir_name='MBSpeech-1.0'):
+        self.keys = keys
+        self.path = os.path.join(os.path.dirname(os.path.realpath(__file__)), dir_name)
+        self.fnames, self.text_lengths, self.texts = read_metadata(os.path.join(self.path, 'metadata.csv'))
+
+    def slice(self, start, end):
+        self.fnames = self.fnames[start:end]
+        self.text_lengths = self.text_lengths[start:end]
+        self.texts = self.texts[start:end]
+
+    def __len__(self):
+        return len(self.fnames)
+
+    def __getitem__(self, index):
+        data = {}
+        if 'texts' in self.keys:
+            data['texts'] = self.texts[index]
+        if 'mels' in self.keys:
+            # (39, 80)
+            data['mels'] = np.load(os.path.join(self.path, 'mels', "%s.npy" % self.fnames[index]))
+        if 'mags' in self.keys:
+            # (39, 80)
+            data['mags'] = np.load(os.path.join(self.path, 'mags', "%s.npy" % self.fnames[index]))
+        if 'mel_gates' in self.keys:
+            data['mel_gates'] = np.ones(data['mels'].shape[0], dtype=np.int)  # TODO: because pre processing!
+        if 'mag_gates' in self.keys:
+            data['mag_gates'] = np.ones(data['mags'].shape[0], dtype=np.int)  # TODO: because pre processing!
+        return data
+
+#
+# simple method to convert mongolian numbers to text, copied from somewhere
+#
+
+
+def number2word(number):
+    digit_len = len(number)
+    digit_name = {1: '', 2: 'мянга', 3: 'сая', 4: 'тэрбум', 5: 'их наяд', 6: 'тунамал'}
+
+    if digit_len == 1:
+        return _last_digit_2_str(number)
+    if digit_len == 2:
+        return _2_digits_2_str(number)
+    if digit_len == 3:
+        return _3_digits_to_str(number)
+    if digit_len < 7:
+        return _3_digits_to_str(number[:-3], False) + ' ' + digit_name[2] + ' ' + _3_digits_to_str(number[-3:])
+
+    digitgroup = [number[0 if i - 3 < 0 else i - 3:i] for i in reversed(range(len(number), 0, -3))]
+    count = len(digitgroup)
+    i = 0
+    result = ''
+    while i < count - 1:
+        result += ' ' + (_3_digits_to_str(digitgroup[i], False) + ' ' + digit_name[count - i])
+        i += 1
+    return result.strip() + ' ' + _3_digits_to_str(digitgroup[-1])
+
+
+def _1_digit_2_str(digit):
+    return {'0': '', '1': 'нэгэн', '2': 'хоёр', '3': 'гурван', '4': 'дөрвөн', '5': 'таван', '6': 'зургаан',
+            '7': 'долоон', '8': 'найман', '9': 'есөн'}[digit]
+
+
+def _last_digit_2_str(digit):
+    return {'0': 'тэг', '1': 'нэг', '2': 'хоёр', '3': 'гурав', '4': 'дөрөв', '5': 'тав', '6': 'зургаа', '7': 'долоо',
+            '8': 'найм', '9': 'ес'}[digit]
+
+
+def _2_digits_2_str(digit, is_fina=True):
+    word2 = {'0': '', '1': 'арван', '2': 'хорин', '3': 'гучин', '4': 'дөчин', '5': 'тавин', '6': 'жаран', '7': 'далан',
+             '8': 'наян', '9': 'ерэн'}
+    word2fina = {'10': 'арав', '20': 'хорь', '30': 'гуч', '40': 'дөч', '50': 'тавь', '60': 'жар', '70': 'дал',
+                 '80': 'ная', '90': 'ер'}
+    if digit[1] == '0':
+        return word2fina[digit] if is_fina else word2[digit[0]]
+    digit1 = _last_digit_2_str(digit[1]) if is_fina else _1_digit_2_str(digit[1])
+    return (word2[digit[0]] + ' ' + digit1).strip()
+
+
+def _3_digits_to_str(digit, is_fina=True):
+    digstr = digit.lstrip('0')
+    if len(digstr) == 0:
+        return ''
+    if len(digstr) == 1:
+        return _1_digit_2_str(digstr)
+    if len(digstr) == 2:
+        return _2_digits_2_str(digstr, is_fina)
+    if digit[-2:] == '00':
+        return _1_digit_2_str(digit[0]) + ' зуу' if is_fina else _1_digit_2_str(digit[0]) + ' зуун'
+    else:
+        return _1_digit_2_str(digit[0]) + ' зуун ' + _2_digits_2_str(digit[-2:], is_fina)