Dataloader custom collate for different input sizes

username: Brenda95X
cooked: <p>Hello,</p>
<p>I’m a fairly new Pytorch user and wondering if anyone could help me with this problem associated with Dataloader.</p>
<p>Here’s a screenshot of my dataframe, inputs are values from ‘y+, index, Re_tau, DU_DY, Y’ column.<br>
<div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/3X/9/5/952b2afe1c139e8312d317fa054087e2e73f0b79.png" data-download-href="https://discuss.pytorch.org/uploads/default/952b2afe1c139e8312d317fa054087e2e73f0b79" title="dataframe"><img src="https://discuss.pytorch.org/uploads/default/original/3X/9/5/952b2afe1c139e8312d317fa054087e2e73f0b79.png" alt="dataframe" data-base62-sha1="lhBD3EfYEtCPLjLZXdn9OhCCyZr" width="690" height="314" data-dominant-color="F0EFEF"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">dataframe</span><span class="informations">1807×823 38.5 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div></p>
<p>Every point in this dataframe, DU_DY &amp; Y always have the same size. However, for different Re_tau values, the size for DU_DY are different (hence, so is the size for Y). And this is where the problem starts in network training. I get a feeling the training data for DU_DY &amp; Y wasn’t split correctly, then checked a few threads and see many CNNs have encountered the same problem and they were solved by calling a custom collate_fn in Dataloader. I attempted to implement my own collate_fn but failed after many tried…</p>
<p>This dataframe ‘df_train’ is first converted to datasets ‘ds_train’, then convert to dataloader.<br>
trainloader = DataLoader(ds_train, batch_size=5,  shuffle=True)</p>
<p>Could anyone point me into the right direction on how to write a collate_fn for this different input size problem? I heard there was a Pytorch preview version solve this problem automatically, sadly not the latest stable version.</p>

username: ptrblck
cooked: <p>Have a look at <a href="https://discuss.pytorch.org/t/dataloader-fails-with-custom-collate-fn/23264/4">this approach</a> posted by <a class="mention" href="/u/vfdev-5">@vfdev-5</a> and let us know, if that helps or not.</p>

username: Brenda95X
cooked: <p>I’ve tried a similar collate_fn before, but doesn’t seem to work if my ds_train is a list that contains tensors.</p>
<p>This is the error meassage I got:<br>
TypeError: expected np.ndarray (got list)</p>
<pre><code class="lang-auto">class ChannelDataset(Dataset):

    def __init__(self, df, input_labels, target_label): # input df is a dataframe
        toArray = lambda x: np.atleast_1d(x)  # convert scalar to array
        toTensor = lambda x: torch.from_numpy(x)  # convert array to tensor
        toFloat = lambda x: x.float() if x.dtype == torch.float64 else x  # convert double to float

        self.df = df.applymap(toArray).applymap(toTensor).applymap(toFloat)

        self.inputs = self.df[input_labels].values
        self.target = self.df[target_label].values

    def __len__(self):
        return len(self.df)

    def __getitem__(self, index):
        inputs = list(self.inputs[index]) # have problem with using torch.cat()
        target = list(self.target[index])[0]

        return inputs, target


class Trainer:
    def _step(self, data_batch, optimizer=None):

        inputs_batch, target_batch = data_batch
        pred_batch = self.model(*inputs_batch)
        loss = self._criterion(pred_batch, target_batch)

        if optimizer is not None:
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

        return loss.item(), len(target_batch)   
   
 
    def fit(self, df_train, df_val, df_test, batch_size=1, print_freq=1, max_epochs=1000, min_epochs=0, earlystopping=False, patience=0):

        self.df_train = df_train
        self.df_val = df_val

        # Convert dataframes to datasets
        ds_train = ChannelDataset(df_train, self.input_labels, self.target_label)
        ds_val = ChannelDataset(df_val, self.input_labels, self.target_label)
        ds_test = ChannelDataset(df_test, self.input_labels, self.target_label)

        # Convert datasets to dataloaders
        trainloader = DataLoader(ds_train, batch_size=batch_size, shuffle=True, collate_fn=lambda batch: [(torch.from_numpy(dp_train[0]), torch.tensor(dp_train[1])) for dp_train in batch])
        valloader = DataLoader(ds_val, batch_size=batch_size, shuffle=False, collate_fn=lambda batch: [(torch.from_numpy(dp[0]), torch.tensor(dp[1])) for dp in batch])
        testloader = DataLoader(ds_test, batch_size=batch_size, shuffle=False, collate_fn=lambda batch: [(torch.from_numpy(dp[0]), torch.tensor(dp[1])) for dp in batch])#collate_fn = my_collate,

        # Initialize loss_history
        if self.loss_history is None:
            self.loss_history = {'train': [], 'val': [], 'test': []}

        while keep_going:  # In each epoch
            # Set model to train mode
            self.model.train()

            # Loop over trainloader
            losses, nums = zip(*[self._step(data_batch, self._optimizer) for data_batch in trainloader])
            train_loss = np.sum(np.multiply(losses, nums)) / np.sum(nums)

            # Set model to eval mode
            self.model.eval()

            # Loop over valloader
            with torch.no_grad():
                losses, nums = zip(*[self._step(data_batch) for data_batch in valloader])
            val_loss = np.sum(np.multiply(losses, nums)) / np.sum(nums)

            # Loop over testloader
            with torch.no_grad():
                losses, nums = zip(*[self._step(data_batch) for data_batch in testloader])
            test_loss = np.sum(np.multiply(losses, nums)) / np.sum(nums)

            # Append losses
            self.loss_history['train'].append(train_loss)
            self.loss_history['val'].append(val_loss)
            self.loss_history['test'].append(test_loss)

</code></pre>

username: ptrblck
cooked: <p>Since <code>ChannelDataset</code> returns <code>lists</code>, <code>torch.from_numpy</code> won’t work and you could try to use <code>torch.tensor(dp[0])</code> instead.</p>

username: Brenda95X
cooked: <p>I tried that earlier, the error for torch.tensor(dp[0]) is:<br>
only one element tensors can be converted to Python scalars</p>
<p>Does this has anything to do with DU_DY and Y has a bracket grouping all data?</p>

username: ptrblck
cooked: <p>I’m not sure what shape and data type <code>self.inputs</code> and <code>self.targets</code> is, but would it work if you remove the <code>list</code> call and pass the arrays directly to the collate function, which will then create the list?</p>

username: Brenda95X
cooked: <p>print(self.inputs) has a shape of (665, 5) and the following structure:</p>
<pre><code class="lang-auto">[[tensor([292.934])  tensor([134], dtype=torch.int32)
  tensor([1.1, 1.2, 1.3, 1.4, ..., 1.153]) # DU_DY, size of 153
  tensor([2.1, 2.2, 2.3, 2.4, ..., 2.153]) # Y, size of 153
  tensor([550.123])] # this Re_tau input determines size for DU_DY &amp; Y
 [tensor([50.433])  tensor([97], dtype=torch.int32)
  tensor([1.1, 1.2, 1.3, 1.4, ..., 1.256]) # DU_DY, size of 256
  tensor([2.1, 2.2, 2.3, 2.4, ..., 2.256]) # Y, size of 256
  tensor([1000.23])] # Re_tau input determines size for DU_DY &amp; Y
 [another training data with different DU_DY &amp; Y size]
 ...
 [another training data with different DU_DY &amp; Y size]]
</code></pre>
<hr>
<p>print(self.target) has a shape of (665, 1) this structure:</p>
<pre><code class="lang-auto">[[tensor([-1.3254])]
 [tensor([-1.9446])]
 ...
 [tensor( [-1.4099])]
 [tensor( [-1.4342])]
 [tensor( [0.])]]
</code></pre>
<hr>
<p>Before getting rid of the list() in class ChannelDataset: element in dataset ds_train has the same list format as ds_train[0]:</p>
<pre><code class="lang-auto">([tensor([292.934]), tensor([134], dtype=torch.int32), tensor([1.1, 1.2, 1.3, 1.4, ..., 1.153]), tensor([2.1, 2.2, 2.3, 2.4, ..., 2.153]), tensor([550.123])], tensor([-1.3254])) 
</code></pre>
<hr>
<p>If I get rid of the list() in ChannelDataset(), the error message upon execution is</p>
<pre><code class="lang-auto">TypeError                                 Traceback (most recent call last)
&lt;ipython-input-4-4e3f094c5224&gt; in &lt;module&gt;
     35 
     36 '''Fit model'''
---&gt; 37 trainer.fit(df_train, df_val, df_test, batch_size=3, print_freq=1, max_epochs=1, earlystopping=False, patience=30)
     38 
     39 scores = {'Train': evaluate(trainer, df_train),

~\Desktop\IACS\Code\Demo\utils_train.py in fit(self, df_train, df_val, df_test, batch_size, print_freq, max_epochs, min_epochs, earlystopping, patience)
    246 
    247             # Loop over trainloader
--&gt; 248             losses, nums = zip(*[self._step(data_batch, self._optimizer) for data_batch in trainloader])
    249             train_loss = np.sum(np.multiply(losses, nums)) / np.sum(nums)
    250 

~\Desktop\IACS\Code\Demo\utils_train.py in &lt;listcomp&gt;(.0)
    246 
    247             # Loop over trainloader
--&gt; 248             losses, nums = zip(*[self._step(data_batch, self._optimizer) for data_batch in trainloader])
    249             train_loss = np.sum(np.multiply(losses, nums)) / np.sum(nums)
    250 

~\Anaconda3\lib\site-packages\torch\utils\data\dataloader.py in __next__(self)
    343 
    344     def __next__(self):
--&gt; 345         data = self._next_data()
    346         self._num_yielded += 1
    347         if self._dataset_kind == _DatasetKind.Iterable and \

~\Anaconda3\lib\site-packages\torch\utils\data\dataloader.py in _next_data(self)
    383     def _next_data(self):
    384         index = self._next_index()  # may raise StopIteration
--&gt; 385         data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
    386         if self._pin_memory:
    387             data = _utils.pin_memory.pin_memory(data)

~\Anaconda3\lib\site-packages\torch\utils\data\_utils\fetch.py in fetch(self, possibly_batched_index)
     45         else:
     46             data = self.dataset[possibly_batched_index]
---&gt; 47         return self.collate_fn(data)

~\Desktop\IACS\Code\Demo\utils_train.py in my_collate(batch)
    198 
    199         def my_collate(batch):
--&gt; 200             output = [(torch.tensor(dp[0]), torch.tensor(dp[1])) for dp in batch]
    201 
    202             return output

~\Desktop\IACS\Code\Demo\utils_train.py in &lt;listcomp&gt;(.0)
    198 
    199         def my_collate(batch):
--&gt; 200             output = [(torch.tensor(dp[0]), torch.tensor(dp[1])) for dp in batch]
    201 
    202             return output

TypeError: can't convert np.ndarray of type numpy.object_. The only supported types are: float64, float32, float16, int64, int32, int16, int8, uint8, and bool.
</code></pre>
<p>I guess now the ds_train is something like</p>
<pre><code class="lang-auto">[(array([tensor([292.934]), tensor([134], dtype=torch.int32), tensor([1.1, 1.2, 1.3, 1.4, ..., 1.153]), tensor([2.1, 2.2, 2.3, 2.4, ..., 2.153]), tensor([550.123])], dtype=object), tensor([-1.3254])),
(array(...)), ...]
</code></pre>
<p>I’m not too sure how to fix the dtype=object issue.</p>