how to use custom dataset in pytorch-lightning module as I am encountering an error "AttributeError: 'str' object has no attribute 'size'"
❓ Questions and Help
how to use custom dataset in pytorch-lightning module as I am encountering an error "AttributeError: 'str' object has no attribute 'size'"?
Code
class CustomDataset(Dataset):
def read_data_set(self):
all_img_files = []
all_labels = []
class_names = os.walk(self.data_set_path).__next__()[1]

for index, class_name in enumerate(class_names):
    label = index
    img_dir = os.path.join(self.data_set_path, class_name)
    img_files = os.walk(img_dir).__next__()[2]
    
    
    for img_file in img_files:
        img_file = os.path.join(img_dir, img_file)
        img = Image.open(img_file)
        if img is not None:
            all_img_files.append(img_file)
            all_labels.append(label)
            
return all_img_files, all_labels, len(all_img_files), len(class_names)

def init(self, data_set_path, transforms=None):
self.data_set_path = data_set_path
self.image_files_path, self.labels, self.length, self.num_classes = self.read_data_set()
self.transforms = transforms
def getitem(self, index):
image = Image.open(self.image_files_path[index])
image = image.convert("RGB")
if self.transforms is not None:
    image = self.transforms(image)
    
return {'image': image, 'label': self.labels[index]}

def len(self):
return self.length
class MNISTClassifier(LightningModule):
def init(self):
super(MNISTClassifier, self).init()
self.layer_1 = torch.nn.Linear(28*28, 128)
self.layer_2 = torch.nn.Linear(128, 256)
self.layer_3 = torch.nn.Linear(256, 10)
def forward(self, x):
batch_size, channels, width, height = x.size()
#(b_s, 1, 28, 28)
x = x.view(batch_size, -1)

#layer1
x = self.layer_1(x)
x = torch.relu(x)

#layer2
x = self.layer_2(x)
x = torch.relu(x)

#layer3
x = self.layer_3()

#probability distribution over labels
x = torch.log_softmax(x, dim = 1)

return x

def cross_entropy_loss(self, logits, labels):
return F.null_loss(logits, labels)
def training_step(self, train_batch, batch_idx):
x, y = train_batch
logits = self.forward(x.size())
loss = self.cross_entropy_loss(logits, y)
logs = {'train_loss': loss}
return {'loss': loss, 'log':logs}

def test_step(self, test_batch, batch_idx):
x, y = test_batch
logits = self.forward(x.size())
loss = self.cross_entropy_loss(logits, y)
logs = {'test_loss:': loss}
return {'val_loss': loss, 'log':logs}

def train_dataloader(self):
dataset = CustomDataset(data_set_path='./files/MNIST/mnist_png/mnist_png/training/', transforms=transforms.ToTensor())
train_loader = DataLoader(dataset, batch_size=32, num_workers=4, shuffle=True)
return train_loader
def test_dataloader(self):
dataset = CustomDataset(data_set_path='files/MNIST/mnist_png/mnist_png/testing/', transforms=transforms.ToTensor())
test_loader = DataLoader(dataset, batch_size=32, num_workers=4)
return test_loader
def configure_optimizers(self):
optimizer = torch.optim.Adam(self.parameters(),lr=1e-3)
return optimizer
model = MNISTClassifier()
trainer = pl.Trainer(gpus=1, max_epochs=1)
trainer.fit(model)
What have you tried?
I am getting an error as follows:
AttributeError Traceback (most recent call last)
in ()
8 model = MNISTClassifier()
9 trainer = pl.Trainer(gpus=1, max_epochs=1)
---> 10 trainer.fit(model)
~/anaconda3/envs/DL_ALL2/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py in fit(self, model, train_dataloader, val_dataloaders)
763
764 elif self.single_gpu:
--> 765 self.single_gpu_train(model)
766
767 elif self.use_tpu: # pragma: no-cover
~/anaconda3/envs/DL_ALL2/lib/python3.6/site-packages/pytorch_lightning/trainer/distrib_parts.py in single_gpu_train(self, model)
490 self.optimizers = optimizers
491
--> 492 self.run_pretrain_routine(model)
493
494 def tpu_train(self, tpu_core_idx, model):
~/anaconda3/envs/DL_ALL2/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py in run_pretrain_routine(self, model)
911
912 # CORE TRAINING LOOP
--> 913 self.train()
914
915 def test(self, model: Optional[LightningModule] = None, test_dataloaders: Optional[DataLoader] = None):
~/anaconda3/envs/DL_ALL2/lib/python3.6/site-packages/pytorch_lightning/trainer/training_loop.py in train(self)
345 # RUN TNG EPOCH
346 # -----------------
--> 347 self.run_training_epoch()
348
349 # update LR schedulers
~/anaconda3/envs/DL_ALL2/lib/python3.6/site-packages/pytorch_lightning/trainer/training_loop.py in run_training_epoch(self)
417 # RUN TRAIN STEP
418 # ---------------
--> 419 _outputs = self.run_training_batch(batch, batch_idx)
420 batch_result, grad_norm_dic, batch_step_metrics, batch_output = _outputs
421
~/anaconda3/envs/DL_ALL2/lib/python3.6/site-packages/pytorch_lightning/trainer/training_loop.py in run_training_batch(self, batch, batch_idx)
594
595 # calculate loss
--> 596 loss, batch_output = optimizer_closure()
597
598 # check if loss or model weights are nan
~/anaconda3/envs/DL_ALL2/lib/python3.6/site-packages/pytorch_lightning/trainer/training_loop.py in optimizer_closure()
558 opt_idx, self.hiddens)
559 else:
--> 560 output_dict = self.training_forward(split_batch, batch_idx, opt_idx, self.hiddens)
561
562 # format and reduce outputs accordingly
~/anaconda3/envs/DL_ALL2/lib/python3.6/site-packages/pytorch_lightning/trainer/training_loop.py in training_forward(self, batch, batch_idx, opt_idx, hiddens)
724 batch = self.transfer_batch_to_gpu(batch, gpu_id)
725 args[0] = batch
--> 726 output = self.model.training_step(*args)
727
728 # TPU support
in training_step(self, train_batch, batch_idx)
36 def training_step(self, train_batch, batch_idx):
37 x, y = train_batch
---> 38 logits = self.forward(x.size())
39 loss = self.cross_entropy_loss(logits, y)
40
AttributeError: 'str' object has no attribute 'size'
What's your environment?

OS: Ubuntu 18.04