Dataloader for multiple files with sliding window

username: kenny
cooked: <p>I am working on a problem where I have multiple CSVs files and I need to read those multiple CSVs one by one with a sliding window. Let’s assume that, one CSV file is having 330 data points and the window size is 32 so we should be having (10*32 = 320) and the last 10 points will be discarded.</p>
<p>I started making a dataset that looks like this but after spending too much time, I am not able to get it working. The current code looks like this,</p>
<pre><code class="lang-auto">class CustomDataset(Dataset):
    def __init__(self, data_folder, window_size):
        self.data_folder = data_folder
        self.data_file_list = [file for file in os.listdir(data_folder)]
        print(self.data_file_list)
        self.window_size = window_size
        
    def __len__(self):
        return len(self.data_file_list[0])

    def __getitem__(self, idx):
        filename = self.data_file_list[idx]
        data, label = read_file(filename)
        return data, label
    
    def read_file(self, filename):
        data = pd.read_csv(filename)
        data = data.drop(["file_name", "class_name"], axis = 1)
        features = data.drop(["class_no"], axis = 1)
        labels = data["class_no"]
        x = [features[index:index+self.window_size].values for index in range(0, len(features))]
        y = [labels[index:index+self.window_size].values for index in range(0, len(labels))]
        
        return x, y
</code></pre>
<p>Note: I can’t merge all these CSV files into one.</p>
<p>I am getting this error,</p>
<blockquote>
<p>TypeError: object of type ‘type’ has no len().</p>
</blockquote>

username: HACS
cooked: <p>Something similar to this?</p>
<p>from sklearn.datasets import load_diabetes, load_iris, load_breast_cancer</p>
<p>import torch<br>
import torch.nn as nn<br>
from torch.utils.data import Dataset, DataLoader</p>
<p>import numpy as np<br>
import pandas as pd</p>
<p>class SimpleDataset(Dataset):<br>
def <strong>init</strong>(self, features, target, name):<br>
self.name = name<br>
self.features = features<br>
self.target = target<br>
self.len = len(target)</p>
<pre><code>def __len__(self):
    return self.len

def __getitem__(self, idx):
    features = self.features[idx]
    target = self.target[idx]
    return features, target
</code></pre>
<p>def dataloader_generator(dataset_list):<br>
for aux in dataset_list:<br>
bunch = aux()<br>
dataset = SimpleDataset(bunch.data, bunch.target, str(aux))<br>
dataloader = DataLoader(dataset=dataset, batch_size=73, drop_last=True)<br>
yield dataloader</p>
<p>dataset_list = [load_diabetes, load_iris, load_breast_cancer]</p>
<p>for epoch in range(2):<br>
print(f’Epoch: {epoch}‘)<br>
for dl in dataloader_generator(dataset_list):<br>
print(f’—&gt; Name: {dl.dataset.name} Cases: {len(dl.dataset)} ’<br>
f’Batches: {len(dl)} Dropped: {len(dl.dataset) - len(dl) * 73}‘)<br>
for features, target in dl:<br>
print(f’------&gt; Batch dimensions: {(features.size(), target.size())}')</p>
<p>Epoch: 0<br>
—&gt; Name: &lt;function load_diabetes at 0x000001CDBE43DA60&gt; Cases: 442 Batches: 6 Dropped: 4<br>
------&gt; Batch dimensions: (torch.Size([73, 10]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 10]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 10]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 10]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 10]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 10]), torch.Size([73]))<br>
—&gt; Name: &lt;function load_iris at 0x000001CDBE43D8B0&gt; Cases: 150 Batches: 2 Dropped: 4<br>
------&gt; Batch dimensions: (torch.Size([73, 4]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 4]), torch.Size([73]))<br>
—&gt; Name: &lt;function load_breast_cancer at 0x000001CDBE43D940&gt; Cases: 569 Batches: 7 Dropped: 58<br>
------&gt; Batch dimensions: (torch.Size([73, 30]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 30]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 30]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 30]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 30]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 30]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 30]), torch.Size([73]))<br>
Epoch: 1<br>
—&gt; Name: &lt;function load_diabetes at 0x000001CDBE43DA60&gt; Cases: 442 Batches: 6 Dropped: 4<br>
------&gt; Batch dimensions: (torch.Size([73, 10]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 10]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 10]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 10]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 10]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 10]), torch.Size([73]))<br>
—&gt; Name: &lt;function load_iris at 0x000001CDBE43D8B0&gt; Cases: 150 Batches: 2 Dropped: 4<br>
------&gt; Batch dimensions: (torch.Size([73, 4]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 4]), torch.Size([73]))<br>
—&gt; Name: &lt;function load_breast_cancer at 0x000001CDBE43D940&gt; Cases: 569 Batches: 7 Dropped: 58<br>
------&gt; Batch dimensions: (torch.Size([73, 30]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 30]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 30]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 30]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 30]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 30]), torch.Size([73]))<br>
------&gt; Batch dimensions: (torch.Size([73, 30]), torch.Size([73]))</p>

username: PaulOppelt
cooked: <p>your  <code>__len__()</code> method should return the total number of features. Right know you are returning the number of files you have in the folder and therefore you will only access one sliding window per file. Unfortunaly there are a lot of problems following from there…The most easy solution is to load all the files and then write all the data into a single list and continue from there. It makes sense to load everything first because loading from files is most of the time very slow.</p>

username: HACS
cooked: <p>Here’s another version of the same suggestion.</p>
<p>#!/usr/bin/env python</p>
<h1>
<a name="coding-utf-8-1" class="anchor" href="#coding-utf-8-1"></a>coding: utf-8</h1>
<h1>
<a name="in1-2" class="anchor" href="#in1-2"></a>In[1]:</h1>
<p>from sklearn.datasets import load_diabetes, load_iris, load_breast_cancer</p>
<p>import torch<br>
import torch.nn as nn<br>
from torch.utils.data import Dataset, DataLoader</p>
<p>import numpy as np<br>
import pandas as pd</p>
<h1>
<a name="in2-3" class="anchor" href="#in2-3"></a>In[2]:</h1>
<h1>
<a name="i-will-store-these-toy-datasets-in-an-hdd-as-if-they-were-large-4" class="anchor" href="#i-will-store-these-toy-datasets-in-an-hdd-as-if-they-were-large-4"></a>I will store these toy datasets in an HDD as if they were large</h1>
<h1>
<a name="files-that-do-not-fit-in-a-pcs-memory-but-fit-in-an-hddsdd-5" class="anchor" href="#files-that-do-not-fit-in-a-pcs-memory-but-fit-in-an-hddsdd-5"></a>files that do not fit in a PC’s memory (but fit in an HDD/SDD).</h1>
<h1>
<a name="target-values-are-in-the-first-column-6" class="anchor" href="#target-values-are-in-the-first-column-6"></a>Target values are in the first column.</h1>
<p>dataset_dict = {<br>
‘diabetes’: load_diabetes,<br>
‘iris’: load_iris,<br>
‘breast_cancer’: load_breast_cancer<br>
}</p>
<p>for name, aux in dataset_dict.items():<br>
aux = aux()<br>
aux = pd.DataFrame(data=np.hstack((aux.target.reshape(-1,1), aux.data)),<br>
columns=[‘Y’] + [‘X’ + str(i) for i in range(aux.data.shape[1])])<br>
aux.to_csv(name + ‘.csv’, index=False)</p>
<h1>
<a name="in80-7" class="anchor" href="#in80-7"></a>In[80]:</h1>
<p>class SimpleDataset(Dataset):<br>
def <strong>init</strong>(self, features, target, size, name):<br>
self.name = name<br>
self.features = features<br>
self.target = target<br>
self.len = size</p>
<pre><code>def __len__(self):
    return self.len

def __getitem__(self, idx):
    features = self.features[idx]
    target = self.target[idx]
    return features, target
</code></pre>
<p>def dataloader_generator(dataset_dict):<br>
max_in_mem = 200  # Max number of instances in memory (g.t.e. to batch sizes)<br>
for name, info in dataset_dict.items():<br>
file_path = info[‘file path’]<br>
batch_size = info[‘batch size’]<br>
norm_chunksize = batch_size * (max_in_mem // batch_size)<br>
with pd.read_csv(file_path, chunksize=norm_chunksize) as reader:<br>
while True:<br>
try:<br>
chunk = next(reader)<br>
aux = chunk.to_numpy()<br>
dataset = SimpleDataset(aux[:, 1:], aux[:, 0], chunk.shape[0], name)<br>
dataloader = DataLoader(dataset=dataset, batch_size=batch_size, drop_last=True)<br>
yield dataloader<br>
except:<br>
break</p>
<h1>
<a name="in81-8" class="anchor" href="#in81-8"></a>In[81]:</h1>
<h1>
<a name="a-dictionary-with-names-file-namespaths-and-batch-sizes-9" class="anchor" href="#a-dictionary-with-names-file-namespaths-and-batch-sizes-9"></a>A dictionary with names, file names/paths, and batch sizes</h1>
<p>dataset_dict = {<br>
‘diabetes’: {<br>
‘file path’:‘diabetes.csv’,<br>
‘batch size’: 200<br>
},<br>
‘iris’: {<br>
‘file path’:‘iris.csv’,<br>
‘batch size’: 150<br>
},<br>
‘breast cancer’: {<br>
‘file path’:‘breast_cancer.csv’,<br>
‘batch size’: 80<br>
}<br>
}</p>
<h1>
<a name="in82-10" class="anchor" href="#in82-10"></a>In[82]:</h1>
<h1>
<a name="running-it-for-1-epoch-11" class="anchor" href="#running-it-for-1-epoch-11"></a>Running it for 1 epoch</h1>
<p>for epoch in range(1):<br>
for dl_idx, dl in enumerate(dataloader_generator(dataset_dict)):<br>
for features, target in dl:<br>
print((dl_idx, dl.dataset.name, dl.dataset.len, features.shape, target.shape))</p>
<p>(0, ‘diabetes’, 200, torch.Size([200, 10]), torch.Size([200]))<br>
(1, ‘diabetes’, 200, torch.Size([200, 10]), torch.Size([200]))<br>
(3, ‘iris’, 150, torch.Size([150, 4]), torch.Size([150]))<br>
(4, ‘breast cancer’, 160, torch.Size([80, 30]), torch.Size([80]))<br>
(4, ‘breast cancer’, 160, torch.Size([80, 30]), torch.Size([80]))<br>
(5, ‘breast cancer’, 160, torch.Size([80, 30]), torch.Size([80]))<br>
(5, ‘breast cancer’, 160, torch.Size([80, 30]), torch.Size([80]))<br>
(6, ‘breast cancer’, 160, torch.Size([80, 30]), torch.Size([80]))<br>
(6, ‘breast cancer’, 160, torch.Size([80, 30]), torch.Size([80]))<br>
(7, ‘breast cancer’, 89, torch.Size([80, 30]), torch.Size([80]))</p>