ghomasHudson (CONTRIBUTOR) commented on 2020-06-23 12:33:22:
A very janky (but working) implementation of  `multitask_dataset.sample()` could be something like this:
```python
import nlp
import torch

class MultiDataset():
    def __init__(self, *args, temperature=2.0, k=1000, maximum=None, scale=1):
        self.datasets = args
        self._dataloaders = {}
        for split in self._get_common_splits():
            split_datasets = [ds[split] for ds in self.datasets]
            mixing_rates = self._calc_mixing_rates(split_datasets,temperature, k, maximum, scale)
            weights = []
            for i in range(len(self.datasets)):
                weights += [mixing_rates[i]]*len(self.datasets[i][split])
            self._dataloaders[split] = torch.utils.data.DataLoader(torch.utils.data.ConcatDataset(split_datasets),
                                                        sampler=torch.utils.data.sampler.WeightedRandomSampler(
                                                            num_samples=len(weights),
                                                            weights = weights,
                                                            replacement=True),
                                                        shuffle=False)

    def _get_common_splits(self):
        '''Finds the common splits present in all self.datasets'''
        min_set = None
        for dataset in self.datasets:
            if min_set != None:
                min_set.intersection(set(dataset.keys()))
            else:
                min_set = set(dataset.keys())
        return min_set


    def _calc_mixing_rates(self,datasets, temperature=2.0, k=1000, maximum=None, scale=1):
       '''Work out the weighting of each dataset based on t and k'''
        mixing_rates = []
        for dataset in datasets:
            rate = len(dataset)
            rate *= scale
            if maximum:
                rate = min(rate, maximum)
            if temperature != 1.0:
                rate = rate ** (1.0/temperature)
            mixing_rates.append(rate)
        return mixing_rates

    def sample(self,n,split):
        batch = []
        for example in self._dataloaders[split]:
            batch.append(example)
            n -= 1
            if n == 0:
                return batch


def flatten(dataset,flatten_fn):
    for k in dataset.keys():
        if isinstance(dataset[k],nlp.Dataset):
            dataset[k] = dataset[k].map(flatten_fn,remove_columns=dataset[k].column_names)

# Squad
def flatten_squad(example):
    return {"source": "squad context: " + example['context'] + " question: " + example['question'],"target":example["answers"]["text"]}
squad = nlp.load_dataset("squad")
flatten(squad,flatten_squad)

# CNN_DM
def flatten_cnn_dm(example):
    return {"source": "cnn_dm: " + example['article'],"target":[example["highlights"]]}
cnn_dm = nlp.load_dataset("cnn_dailymail", "3.0.0")
flatten(cnn_dm,flatten_cnn_dm)

multitask_dataset = MultiDataset(squad, cnn_dm)
batch = multitask_dataset.sample(100,"train")
```

There's definitely a more sensible way than embedding `DataLoader`s inside.