Repository: speechbrain/speechbrain

Issue:
minloglevel issue with SentencePiece
Hi @aheba,

It's not the first time that I face this problem, and it's a big one on Cedar (could be the case on other clusters). SentencePiece 0.1.92 cannot be installed (as it is yanked according to the official GitHub, the version has some big regression bugs). Problem: 0.1.91 doesn't accept minloglevel as an argument raising an error 

**OSError: Not found: unknown field name "minloglevel" in TrainerSpec.**
I suppose that we have to remove this option so it always work, but the logs are terrible ... any ideas ?


Discussion:

Resolving patch:
diff --git a/speechbrain/data_io/datasets.py b/speechbrain/data_io/datasets.py
new file mode 100644
--- /dev/null
+++ b/speechbrain/data_io/datasets.py
@@ -0,0 +1,59 @@
+from torch.utils.data import Dataset
+
+
+class SegmentedDataset(Dataset):
+
+    """
+    SegmentedDataset handles pre-segmented datasets.
+
+    It takes in input a dict in which each entry is a single example.
+    The example in its turn is represented by using a dict:
+    e.g. {"wav_file": "/path/to/helloworld.wav" "words": ["hello", "world"]}.
+
+    in data_fields a list of elements which one wants to be returned by this dataset class __getitem__
+    method must be specifies e.g. data_fields = ["wav_file", "words"].
+    Corresponding data_transformation can be specified for each element we want to return.
+    Note that transformations include also reading data from a file:
+    e.g. {"wav_file": read_wav} where read_wav is a suitable function we provide in data_io.py.
+
+    Finally the specified elements are tranformed an returned from __getitem__
+    in a dict where the keys corresponds to the data_fields entried e.g. "wav_file".
+
+
+     ---------
+    Examples : dict
+        Dictionary containing single examples (e.g. utterances).
+    data_fields : (list, tuple)
+        The class to use for updating the modules' parameters.
+    data_transforms : dict
+        Dictionary where data transforms for each field is specified.
+    """
+
+    def __init__(
+        self, examples: dict, data_fields: (list, tuple), data_transforms=None,
+    ):
+
+        self.data_fields = data_fields
+        self.data_transforms = data_transforms
+
+        assert isinstance(self.data_transforms, dict)
+        for k in self.data_transforms.keys():
+            assert callable(
+                self.data_transforms[k]
+            ), "Each element in data_transforms dict must be callable"
+        self.examples = examples
+        self.ex_ids = list(self.examples.keys())
+
+    def __len__(self):
+        return len(self.ex_ids)
+
+    def __getitem__(self, item):
+        ex_id = self.ex_ids[item]
+        c_ex = self.examples[ex_id]
+        out = {"id": ex_id}
+
+        for k in c_ex.keys():
+            if k in self.data_fields:
+                out[k] = self.data_transforms[k](c_ex[k])
+
+        return out
diff --git a/speechbrain/data_io/utils.py b/speechbrain/data_io/utils.py
--- a/speechbrain/data_io/utils.py
+++ b/speechbrain/data_io/utils.py
@@ -1,274 +1,99 @@
-import os
 import torch
-from ruamel import yaml
-from collections.abc import MutableMapping
-from copy import copy
 
 
-class DataCollection(MutableMapping):
-    # TODO not still sure about this
-    def __init__(self, path_to_yaml):
-
-        with open(path_to_yaml, "r") as f:
-            self.datacol = yaml.safe_load(f)
-
-    def __getitem__(self, key):
-        return self.datacol[key]
-
-    def __delitem__(self, key):
-        del self.datacol[key]
-
-    def __setitem__(self, key, value):
-        if key in self:
-            del self[self[key]]
-        if value in self:
-            del self[value]
-        self.datacol[key] = value
-        self.datacol[value] = key
-
-    def __iter__(self):
-        return iter(self.datacol)
-
-    def __len__(self):
-        return len(self.datacol)
-
-    # def __repr__(self):
-    #   return f"{type(self).__name__}({self.datacol})
-
-    def sanity_check(self):
-        dataset_sanity_check(self)
-
-    def to_ASR_format(self):
-        pass
-
-    def filter(self):
-        pass
-
-    def sort(self):
-        pass
-
-    @staticmethod
-    def merge(list_of_datacollections):
-        pass
-
-
-def dataset_sanity_check(dataset_splits):
-
-    # dataset_structs is a list of dataset_struct
-    # each dataset struct is a dict-based hierarchical annotation
-    # we check for consistency between all of these e.g. between train.yaml, dev.yaml and test.yaml
-
-    assert len(dataset_splits) > 0, "provided list of dataset splits is empty"
-
-    # we take first supervision of first structure
-    first_key = list(dataset_splits[0].keys())[0]
-    first_sup_keys = copy(
-        list(dataset_splits[0][first_key]["supervision"][0].keys())
-    )
-
-    for d_split in dataset_splits:
-        for data_obj_id in d_split.keys():
-            c_obj = d_split[data_obj_id]
-            # we check that waveforms have at least one file, channels, lengths, and samplerate
-            assert all(
-                [
-                    k in c_obj["waveforms"].keys()
-                    for k in ["files", "channels", "samplerate", "lengths"]
-                ]
-            ), "Waveforms entries should have always files, channels, samplerate and length keys. "
-            # assert are not empty and are of proper type
-            assert isinstance(c_obj["waveforms"]["files"], list)
-            assert isinstance(c_obj["waveforms"]["files"][0], str)
-            assert isinstance(c_obj["waveforms"]["channels"], list)
-            assert isinstance(c_obj["waveforms"]["channels"][0], list)
-            assert isinstance(c_obj["waveforms"]["lengths"], list)
-            assert isinstance(c_obj["waveforms"]["lengths"][0], int)
-            assert isinstance(c_obj["waveforms"]["samplerate"], int)
-
-            # assert files exists
-            for f in c_obj["waveforms"]["files"]:
-                assert os.path.exists(f), "{} does not exist".format(f)
-
-            # we should also check for external paths here how do we specify that an entry is a path ?
-
-            # check if there are any duplicates in supervision in the same data_obj
-            seen = set()
-            for sup in c_obj["supervision"]:
-                t = tuple(
-                    (k, str(v)) for (k, v) in sup.items()
-                )  # tuplefy lists to make em hashable
-                if t not in seen:
-                    seen.add(t)
-                else:
-                    raise KeyError(
-                        "Supervision for data object ID {} contains duplicates please remove them".format(
-                            data_obj_id
-                        )
-                    )
-
-            assert (
-                len(c_obj["supervision"]) > 0
-            ), "At least one supervision should be specified for each data obj"
-
-            for sup in c_obj["supervision"]:
-                assert (
-                    list(sup.keys()) == first_sup_keys
-                ), "All supervision must have same fields within a data object and must be ordered in same way"
-                # assert all supervisions in all
-
-                for sup_name in sup.keys():
-                    assert isinstance(
-                        sup[sup_name],
-                        (tuple, list, float, int, bool, str, dict),
-                    ), "Format not supported"
-
-    for d_split in dataset_splits:
-        for data_obj_id in d_split.keys():
-            c_obj = d_split[data_obj_id]
-            for sup in c_obj["supervision"]:
-                # dataset have same supervisions
-                # if start and stop are not specified we assume that all file is used
-                # we make it explicit and take start and stop from waveforms.
-                if not all(k in sup.keys() for k in ("start", "stop")):
-                    sup["start"] = 0  # we modify it in place
-                    sup["stop"] = min(c_obj["waveforms"]["lengths"])
-
-                elif all(k in sup.keys() for k in ("start", "stop")):
-                    pass
-                else:
-                    raise EnvironmentError(
-                        "You can't specify only start or stop. Either specify both or none of the two"
-                    )
-
-        # how we specify external path dependencies ?
-
-
-def to_ASR_format(dataset):
-    """
-    Converts general dataset format to ASR format where we have a list of unique utterances.
+def pad_right_to(
+    tensor: torch.Tensor,
+    target_shape: (list, tuple),
+    mode="constant",
+    value=0.0,
+):
     """
+    This function takes a torch tensor of arbitrary shape and pads it to target
+    shape by appending values on the right.
+
+    Parameters
+    ----------
+    tensor: input torch tensor
+        Input tensor whose dimension we need to pad.
+    target_shape: (list, tuple)
+        Target shape we want for the target tensor its len must be equal to tensor.ndim
+    mode: str
+        Pad mode, please refer to torch.nn.functional.pad documentation.
+    value: float
+        Pad value
+
+    Returns
+    -------
+    tensor: torch.Tensor
+        Padded tensor
+    valid_vals: list
+        List containing proportion for each dimension of original, non-padded values
 
-    utterances = []
-    for data_obj_id in dataset.keys():
-        for supervision in dataset[data_obj_id]["supervision"]:
-            # we "reverse" the format
-            utterances.append(
-                {
-                    "supervision": supervision,
-                    "waveforms": dataset[data_obj_id]["waveforms"],
-                }
-            )
-
-    return utterances
-
-
-class CategoricalEncoder:
-    def __init__(self, data_collections: (list, dict), supervision: str):
-        if isinstance(data_collections, dict):
-            data_collections = [data_collections]
-
-        all_labs = set()
-        for data_coll in data_collections:
-            for data_obj_key in data_coll:
-                data_obj = data_coll[data_obj_key]
-                for sup in data_obj["supervision"]:
-                    for sup_key in sup.keys():
-                        if sup_key == supervision:
-                            if isinstance(sup[sup_key], (list, tuple)):
-                                all_labs.update(set(sup[sup_key]))
-                            elif isinstance(sup[sup_key], (str)):
-                                all_labs.add(sup[sup_key])
-                            else:
-                                raise NotImplementedError
-
-        all_labs = sorted(list(all_labs))  # sort alphabetically just in case
-
-        self.lab2indx = {key: index for index, key in enumerate(all_labs)}
-        self.indx2lab = {key: index for key, index in enumerate(all_labs)}
-
-    def encode(self, x, dtype=torch.long):
-        # TODO handle numpy arrays maybe ?
-        if isinstance(x, (tuple, list)):
-            # then we will return
-            labels = torch.zeros(
-                (len(self.lab2indx.keys()), len(x)), dtype=dtype
-            )
-            for i, elem in enumerate(x):
-                labels[self.lab2indx[elem], i] = 1
-        else:
-            labels = torch.zeros((len(self.lab2indx.keys()), 1), dtype=dtype)
-
-        return labels
-
-    def decode(self, x: torch.Tensor):
-        # labels are of shape num_classes or num_classes, lengthofseq
-        if x.ndim == 1:
-            indx = torch.argmax(x)
-            decoded = self.indx2lab[indx]
-            return decoded
-
-        elif x.ndim == 2:
-            decoded = []
-            indexes = torch.argmax(x, 0)
-            for time_step in range(len(indexes)):
-                decoded.append(self.indx2lab[indexes[time_step]])
-            return decoded
-
-        elif x.ndim == 3:  # batched tensor b, classes, steps
-            decoded = []
-            for batch in x.shape[0]:
-                c_batch = []
-                indexes = torch.argmax(x[batch], 0)
-                for time_step in range(len(indexes)):
-                    c_batch.append(self.indx2lab[indexes[time_step]])
-                decoded.append(c_batch)
-            return decoded
-        else:
-            raise NotImplementedError
-
+    """
+    assert len(target_shape) == tensor.ndim
 
-def replace_entries(data_coll, replacements_dict):
+    pads = []
+    valid_vals = []
+    i = len(target_shape) - 1
+    j = 0
+    while i >= 0:
+        assert (
+            target_shape[i] >= tensor.shape[i]
+        ), "Target shape must be >= original shape for every dim"
+        pads.extend([0, target_shape[i] - tensor.shape[i]])
+        valid_vals.append(tensor.shape[j] / target_shape[j])
+        i -= 1
+        j += 1
 
-    for data_obj_key in data_coll:
-        data_obj = data_coll[data_obj_key]
-        for sup in [*data_obj["supervision"], data_obj["waveforms"]]:
-            for sup_key in sup.keys():
-                if sup_key in replacements_dict.keys():
-                    if isinstance(sup[sup_key], (str)):
-                        mapping = replacements_dict[sup_key]
-                        for map_key in mapping.keys():
-                            # we replace in place
-                            sup[sup_key] = sup[sup_key].replace(
-                                map_key, mapping[map_key]
-                            )
-                    elif isinstance(sup[sup_key], (list, tuple)):
-                        assert isinstance(
-                            sup[sup_key][0], str
-                        ), "Replacements supported only for str type, and unidimensional lists"
-                        mapping = replacements_dict[sup_key]
-                        for map_key in mapping.keys():
-                            for indx in range(len(sup[sup_key])):
-                                sup[sup_key][indx] = sup[sup_key][indx].replace(
-                                    map_key, mapping[map_key]
-                                )
-                    else:
-                        raise NotImplementedError
-                    # check if we have to replace it
+    tensor = torch.nn.functional.pad(tensor, pads, mode=mode, value=value)
 
-    return data_coll
+    return tensor, valid_vals
 
 
-def get_windowed_examples(dataset):
-    """
-    get examples for diarization, vad and other speech labelling applications
-    from long files by using a sliding window with overlap.
+def batch_pad_right(tensors: list, mode="constant", value=0.0):
     """
-    pass
-
-
-def filter_supervision(dataset):
-    pass
+    Given a list of torch tensors it batches them together by padding to the right
+    on each dimension in order to get same length for all.
+
+    Parameters
+    ----------
+    tensors: list
+        List of tensor we wish to pad together.
+    mode: str
+        Padding mode see torch.nn.functional.pad documentation.
+    value: float
+        Padding value.
+
+    Returns
+    -------
+    tensor: torch.Tensor
+        Padded tensor
+    valid_vals: list
+        List containing proportion for each dimension of original, non-padded values
 
+    """
 
-def filter_waveforms(dataset):
-    pass
+    assert len(tensors), "Tensors list must not be empty"
+    if len(tensors) == 1:
+        return tensors[0].unsqueeze(0), [[1.0 for x in range(tensors[0].ndim)]]
+    assert any(
+        [tensors[i].ndim == tensors[0].ndim for i in range(1, len(tensors))]
+    ), "All tensors must have same number of dimensions"
+
+    # we gather the max length for each dimension
+    max_shape = []
+    for dim in range(tensors[0].ndim):
+        max_shape.append(max([x.shape[dim] for x in tensors]))
+
+    batched = []
+    valid = []
+    for t in tensors:
+        padded, valid_percent = pad_right_to(
+            t, max_shape, mode=mode, value=value
+        )
+        batched.append(padded)
+        valid.append(valid_percent)
+
+    batched = torch.stack(batched)
+
+    return batched, valid


Test patch: