Torch Sampler ImbalancedDataSampler

NolimitDougie: <pre><code class="lang-auto">import numpy as np
import pandas as pd
import os
import pickle
from glob import glob
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
from torchvision import datasets, transforms
from torch.utils.data import Dataset, DataLoader, TensorDataset
import torch.nn as nn
import torch
import torchvision
import seaborn as sns
from tqdm import tqdm
from PIL import Image
from itertools import chain
import torch.nn.functional as F
import torch.optim as optim
from sklearn.utils import resample
import torch.optim.lr_scheduler as lr_scheduler
from sklearn.metrics import f1_score, accuracy_score, precision_score, recall_score, roc_auc_score, \
    multilabel_confusion_matrix, roc_curve, auc, classification_report
from torchsampler import ImbalancedDatasetSampler

# Device configuration GPU support for MAC
if torch.backends.mps.is_available():
    mps_device = torch.device("mps")
else:
    print("MPS device not found.")

# Paths to Images and DataEntry file
all_xray_df = pd.read_csv('NihXrayData/Data_Entry_2017_v2020.csv')
allImagesGlob = glob('NihXrayData/images*/images/*.png')
# eof

all_image_paths = {os.path.basename(x): x for x in
                   allImagesGlob}
# print('Scans found:', len(all_image_paths), ', Total Headers', all_xray_df.shape[0])
all_xray_df['path'] = all_xray_df['Image Index'].map(all_image_paths.get)
all_xray_df.sample(3)

# # Data Pre Processing ####
# # Simplifying to 15 primary classes (adding No Finding as the 15th class)
condition_labels = ['Atelectasis', 'Consolidation', 'Infiltration', 'Pneumothorax', 'Edema', 'Emphysema', 'Fibrosis',
                    'Effusion', 'Pneumonia', 'Pleural_Thickening',
                    'Cardiomegaly', 'Nodule', 'Mass', 'Hernia', 'No Finding']
for label in condition_labels:
    all_xray_df[label] = all_xray_df['Finding Labels'].map(lambda result: 1.0 if label in result else 0)
all_xray_df.head(20)

all_xray_df['disease_vec'] = all_xray_df.apply(lambda target: [target[condition_labels].values], 1).map(
    lambda target: target[0])

all_xray_df.head()

print(all_xray_df[condition_labels].sum())

train_df, test_df = train_test_split(all_xray_df, test_size=0.30, random_state=2020)


class XrayDataset(torch.utils.data.Dataset):
    def __init__(self, data_frame, transform=None):
        self.data_frame = data_frame
        self.transform = transform

    def __getitem__(self, idx):
        row = self.data_frame.iloc[idx]
        address = row['path']
        data = Image.open(address).convert('RGB')
        label = np.array(row['disease_vec'], dtype=np.float32)

        if self.transform:
            data = self.transform(data)

        return data, torch.FloatTensor(label)

    def __len__(self):
        return len(self.data_frame)


# Define data augmentation for training
train_transform = transforms.Compose([
    # transforms.RandomResizedCrop(224),
    transforms.Resize(224),
    transforms.RandomHorizontalFlip(),
    transforms.RandomVerticalFlip(),
    transforms.RandomRotation(10),
    transforms.RandomGrayscale(p=0.4),
    transforms.ToTensor(),
    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
])

# Data Sets
train_dataset = XrayDataset(train_df, transform=train_transform)
test_dataset = XrayDataset(test_df, transform=transforms.Compose([
    transforms.Resize(224),
    transforms.ToTensor(),
    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
]))

trainSampler = ImbalancedDatasetSampler(train_dataset)
# Data Loaders
train_loader = DataLoader(
    train_dataset,
    batch_size=64,
    num_workers=0,
    sampler=trainSampler,
    shuffle=True,
)

test_loader = torch.utils.data.DataLoader(
    test_dataset,
    batch_size=64,
    num_workers=0,
    shuffle=False,
)
</code></pre>
<pre><code class="lang-auto">Traceback (most recent call last):
  File "/Users/dougietownsell/PycharmProjects/Independent-Study/Xray.py", line 101, in &lt;module&gt;
    trainSampler = ImbalancedDatasetSampler(train_dataset)
  File "/Users/dougietownsell/PycharmProjects/Independent-Study/venv/lib/python3.9/site-packages/torchsampler/imbalanced.py", line 37, in __init__
    df["label"] = self._get_labels(dataset) if labels is None else labels
  File "/Users/dougietownsell/PycharmProjects/Independent-Study/venv/lib/python3.9/site-packages/torchsampler/imbalanced.py", line 61, in _get_labels
    return dataset.get_labels()
AttributeError: 'XrayDataset' object has no attribute 'get_labels'
</code></pre>
<p>Hello I’m trying to use the ImbalancedDataSampler but I keep getting this error not sure how to fix it</p>

probit2011: <p>Judging by the example on the <a href="https://pypi.org/project/torchsampler/" rel="noopener nofollow ugc">TorchSampler project webpage</a>, you need to instantiate the sampler directly inside the DataLoader instantiation ( <code>DataLoader(train_dataset, ..., sampler=ImbalancedDatasetSampler(train_dataset))</code> ), not outside. Not easy to verify whether this is the case though, as the link to the project’s homepage returns a 404 message. In any case, check how the <code>_get_labels</code> function is implemented in <code>imbalanced.py</code>.</p>

yiftach: <p>It seems like the imbalanced sampler assumes the dataset object you supply has a <code>get_labels()</code> function, which the <code>XrayDataset</code> you use doesn’t have. You’ll have to check the torchsampler docs to see what the API is, then add an appropriate implementation to <code>XRayDataset</code>’.</p>

meirashaf: <p>Just adding a reference</p>
<aside class="onebox githubissue" data-onebox-src="https://github.com/ufoym/imbalanced-dataset-sampler/issues/34">
  <header class="source">

      <a href="https://github.com/ufoym/imbalanced-dataset-sampler/issues/34" target="_blank" rel="noopener nofollow ugc">github.com/ufoym/imbalanced-dataset-sampler</a>
  </header>

  <article class="onebox-body">
    <div class="github-row">
  <div class="github-icon-container" title="Issue" data-github-private-repo="false">
	  <svg width="60" height="60" class="github-icon" viewBox="0 0 14 16" aria-hidden="true"><path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"></path></svg>
  </div>

  <div class="github-info-container">
    <h4>
      <a href="https://github.com/ufoym/imbalanced-dataset-sampler/issues/34" target="_blank" rel="noopener nofollow ugc">'MyDataset' object has no attribute 'get_labels'</a>
    </h4>

    <div class="github-info">
      <div class="date">
        opened <span class="discourse-local-date" data-format="ll" data-date="2021-07-02" data-time="07:12:43" data-timezone="UTC">07:12AM - 02 Jul 21 UTC</span>
      </div>

        <div class="date">
          closed <span class="discourse-local-date" data-format="ll" data-date="2021-07-11" data-time="23:53:49" data-timezone="UTC">11:53PM - 11 Jul 21 UTC</span>
        </div>

      <div class="user">
        <a href="https://github.com/kuri54" target="_blank" rel="noopener nofollow ugc">
          <img alt="kuri54" src="https://avatars.githubusercontent.com/u/40049003?v=4" class="onebox-avatar-inline" width="20" height="20">
          kuri54
        </a>
      </div>
    </div>

    <div class="labels">
    </div>
  </div>
</div>

  <div class="github-row">
    <p class="github-body-container">When I try to use my own Dataset class, I get the error `'MyDataset' object has <span class="show-more-container"><a href="" rel="noopener" class="show-more">…</a></span><span class="excerpt hidden">no attribute 'get_labels'` and cannot proceed.

The content of the Dataloader is as follows, and there is nothing strange about it.
It processes the image data and label data in .npz format.
```
class MyDataset(data.Dataset):
    def __init__(self, images, labels, transform=None):
        self.images = images
        self.labels = labels
        self.transform = transform

    def __len__(self):
        return len(self.images)

    def __getitem__(self, index):
        image = self.images[index]
        label = self.labels[index]

        if self.transform is not None:
            image = self.transform(image=image)["image"]

        return image, label
```

```
train_dataset = MyDataset(train_imgs, train_labels, transform=transform)
train_dataloader = torch.utils.data.DataLoader(train_dataset,
                                               sampler=ImbalancedDatasetSampler(train_dataset),
                                               batch_size= batch_size,
                                               shuffle=True,
                                               num_workers=2)
```

Is there something wrong with the code?
I don't think it's a typo.

How can I fix it so that it works correctly?</span></p>
  </div>

  </article>

  <div class="onebox-metadata">
    
    
  </div>

  <div style="clear: both"></div>
</aside>