Problem while training custom dataset

mchowdhu: <p>I have around 4000 .npy files which contain two different images namely ‘R’ &amp; ‘L’. I also have another data .csv file which contain list of data ID,SIDE,Lable.I used these two files(npy,csv) to process my custom dataset using the reference<a href="https://pytorch.org/tutorials/beginner/data_loading_tutorial.html#dataset-class" rel="nofollow noopener">tutorial</a>.My aim is image classification using CNN.<br>
here is my code</p>
<pre><code class="lang-auto">root_dirA= '/home/fatema/Downloads/homework/datasetA/p26crops/'


csvroot = '/home/fatema/Downloads/homework/datafile.csv'
#read the csv file
classfile =pd.read_csv(csvroot)

# remove the NA value from the file
df = classfile.dropna(how='any',axis=0)
# Dropout the duplicates of number from ID and Side

data = df.drop_duplicates(subset=['ID', 'SIDE'])
#reset the index
Data = data.reset_index(drop=True)
#make a class for data
class AOIDataset(Dataset):
    def __init__(self, csv_file,root_dir, transform=None):
               # self.data = pd.read_csv(csv_file, header=None)
               self.data = csv_file
               self.root_dir = root_dir
               self.transform = transform



    def __len__(self):
       return len(self.data)

    def __getitem__(self,idx):
        img_name = os.path.join(self.root_dir,
                               self.data['ID'][idx])
        patches, p_id = np.load(img_name)

        img_class = self.data.iloc[idx,2]

        side = self.data.iloc[idx,1]

        if (self.data['SIDE'][idx,:1]==1):
        # #if (self.data.iloc[idx,0]) ==1:
         image = np.array(patches['R'].astype('uint8'), 'L')                                              #[image['R':side]|image['L':side]]
        else:
         image = np.array(patches['L'].astype('uint8'), 'L')

         if self.transform is not None:
            image = self.transform(image)




        sample = {'image': image, 'grade':img_class}


        return sample


if __name__ == '__main__':
# Define transforms (1)
 #trans = transforms.Compose([ transforms.ToTensor()])
    # Call the dataset

 train_data =AOIDataset(train_df,root_dirA)


</code></pre>
<p>here is my training function code</p>
<pre><code class="lang-auto">def train_model(epochs):
    model.train() #set the model to training mode
    for epoch in range(epochs):
        losses = []
        num_times=0
        closs = 0
        for i,batch in enumerate(train_loader,0):
            image , grade = batch
            image=image.unsqueeze(1).type(torch.FloatTensor)
            prediction = model(image)
            loss = costFunction(prediction,grade)
            closs += loss.item()
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            #Track every 100th loss
            if i%50 == 0:
                losses.append(loss.item())
                num_times = num_times + 1

            #print every 1000th time
            if i%50 == 0:
                print('[%d  %d] loss: %.4f'% (epoch+1,i+1,closs/50))
                closs = 0
        #Calculate the accuracy and save the model state
                accuracy()
                #Plot the graph of loss with iteration
                plt.plot([i for i in range(num_times)],losses,label='epoch'+str(epoch))
                plt.legend(loc=1,mode='expanded',shadow=True,ncol=2)
                plt.show()
def accuracy():
     model.eval() #set the model to evaluation mode
     #Calculate the overall performance of the network
     correctHits=0
     total=0
     accuracy=0
     for batches in val_loader:
         image,grade = batches
         image=image.unsqueeze(1).type(torch.FloatTensor)
         prediction = model(image)
         _,prediction = torch.max(prediction.data,1)  #returns max as well as its index
         total += grade.size(0)
         correctHits += (prediction==grade).sum().item()
         accuracy = (correctHits/total)*100


         print('Accuracy = '+str(accuracy))

if __name__ == '__main__':
    train_model(1)


The error that I got is here
</code></pre>
<p>Traceback (most recent call last):<br>
File “/home/fatema/miniconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py”, line 3325, in run_code<br>
exec(code_obj, self.user_global_ns, self.user_ns)<br>
File “”, line 1, in <br>
runfile(’/home/fatema/Downloads/practice1.py’, wdir=’/home/fatema/Downloads’)<br>
File “/home/fatema/Downloads/pycharm-professional-2019.1.3/pycharm-2019.1.3/helpers/pydev/_pydev_bundle/pydev_umd.py”, line 197, in runfile<br>
pydev_imports.execfile(filename, global_vars, local_vars)  # execute the script<br>
File “/home/fatema/Downloads/pycharm-professional-2019.1.3/pycharm-2019.1.3/helpers/pydev/_pydev_imps/_pydev_execfile.py”, line 18, in execfile<br>
exec(compile(contents+"\n", file, ‘exec’), glob, loc)<br>
File “/home/fatema/Downloads/practice1.py”, line 223, in <br>
train_model(1)<br>
File “/home/fatema/Downloads/practice1.py”, line 180, in train_model<br>
for i,batch in enumerate(train_loader,0):<br>
File “/home/fatema/miniconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py”, line 560, in <strong>next</strong><br>
batch = self.collate_fn([self.dataset[i] for i in indices])<br>
File “/home/fatema/miniconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py”, line 560, in <br>
batch = self.collate_fn([self.dataset[i] for i in indices])<br>
File “/home/fatema/miniconda3/lib/python3.6/site-packages/torch/utils/data/dataset.py”, line 107, in <strong>getitem</strong><br>
return self.dataset[self.indices[idx]]<br>
File “/home/fatema/Downloads/practice1.py”, line 64, in <strong>getitem</strong><br>
self.data[‘ID’][idx])<br>
File “/home/fatema/miniconda3/lib/python3.6/posixpath.py”, line 94, in join<br>
genericpath._check_arg_types(‘join’, a, *p)<br>
File “/home/fatema/miniconda3/lib/python3.6/genericpath.py”, line 149, in _check_arg_types<br>
(funcname, s.<strong>class</strong>.<strong>name</strong>)) from None<br>
TypeError: join() argument must be str or bytes, not ‘int64’</p>
<pre><code class="lang-auto"></code></pre>

Mazhar_Shaikh: <p>Hi mchowdhu,<br>
Looks like the error is due to a type mismatch in the os.path.join() function call.</p>
<aside class="quote no-group quote-modified" data-username="mchowdhu" data-post="1" data-topic="51769">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/letter_avatar_proxy/v4/letter/m/4af34b/48.png" class="avatar"> mchowdhu:</div>
<blockquote>
<p>img_name = os.path.join(self.root_dir, self.data[‘ID’][idx])</p>
</blockquote>
</aside>
<p>The correct way is shown here:</p>
<pre><code class="lang-auto">img_name = os.path.join(self.root_dir, str(self.data['ID'][idx]))
</code></pre>
<p>Hope this solves the current error.</p>

mchowdhu: <p><a class="mention" href="/u/mazhar_shaikh">@Mazhar_Shaikh</a>, I changed the following line,<br>
`    def <strong>getitem</strong>(self,idx):<br>
img_name = os.path.join(self.root_dir, str(self.data[‘ID’][idx]))<br>
patches, p_id = np.load(img_name)</p>
<pre><code>    img_class = self.data.iloc[idx,2]

    side = self.data.iloc[idx,1]

    if (self.data['SIDE'][idx,:1]==1):
    # #if (self.data.iloc[idx,0]) ==1:
     image = np.array(patches['R'].astype('uint8'), 'L')                                              #[image['R':side]|image['L':side]]
    else:
     image = np.array(patches['L'].astype('uint8'), 'L')

     if self.transform is not None:
        image = self.transform(image)




    sample = {'image': image, 'grade':img_class}


    return sample
</code></pre>
<p>`<br>
now i am getting the new error</p>
<pre><code class="lang-auto">runfile('/home/fatema/Downloads/practice1.py', wdir='/home/fatema/Downloads')
Traceback (most recent call last):
  File "/home/fatema/miniconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3325, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "&lt;ipython-input-2-052e1fda1dc0&gt;", line 1, in &lt;module&gt;
    runfile('/home/fatema/Downloads/practice1.py', wdir='/home/fatema/Downloads')
  File "/home/fatema/Downloads/pycharm-professional-2019.1.3/pycharm-2019.1.3/helpers/pydev/_pydev_bundle/pydev_umd.py", line 197, in runfile
    pydev_imports.execfile(filename, global_vars, local_vars)  # execute the script
  File "/home/fatema/Downloads/pycharm-professional-2019.1.3/pycharm-2019.1.3/helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile
    exec(compile(contents+"\n", file, 'exec'), glob, loc)
  File "/home/fatema/Downloads/practice1.py", line 98, in &lt;module&gt;
    sample = train_data[i]
  File "/home/fatema/Downloads/practice1.py", line 64, in __getitem__
    patches, p_id = np.load(img_name)
  File "/home/fatema/miniconda3/lib/python3.6/site-packages/numpy/lib/npyio.py", line 422, in load
    fid = open(os_fspath(file), "rb")
FileNotFoundError: [Errno 2] No such file or directory: '/home/fatema/Downloads/homework/datasetA/p26crops/9000099'

</code></pre>

Mazhar_Shaikh: <p>Please also add the file extension.</p>
<pre><code class="lang-auto">img_name = os.path.join(self.root_dir, str(self.data[‘ID’][idx])+'.npy')
</code></pre>

mchowdhu: <aside class="quote no-group quote-modified" data-username="Mazhar_Shaikh" data-post="4" data-topic="51769">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/mazhar_shaikh/48/14501_2.png" class="avatar"> Mazhar_Shaikh:</div>
<blockquote>
<p>os.path.join(self.root_dir, str(self.data[‘ID’][idx])+‘.npy’</p>
</blockquote>
</aside>
<p>I  have modified the line</p>
<pre><code class="lang-auto">   def __getitem__(self,idx):
       img_name = os.path.join(self.root_dir, str(self.data['ID'][idx])+'.npy')


       patches, p_id = np.load(img_name,allow_pickle= True)

       img_class = (self.data.iloc[idx,2])

       side = self.data.iloc[idx,1]

       if (self.data.iloc[idx,1]==1):
       # #if (self.data.iloc[idx,0]) ==1:
        image = np.array(patches['R'].astype('uint8'), 'L')                                              #[image['R':side]|image['L':side]]
       else:
        image = np.array(patches['L'].astype('uint8'), 'L')

        if self.transform is not None:
           image = self.transform(image)




       sample = {'image': image, 'grade':img_class}


       return sample

</code></pre>
<p>new error has occurred. <a class="mention" href="/u/mazhar_shaikh">@Mazhar_Shaikh</a> ,<br>
here is the error</p>
<pre><code class="lang-auto">runfile('/home/fatema/Downloads/practice1.py', wdir='/home/fatema/Downloads')
Traceback (most recent call last):
  File "/home/fatema/miniconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3325, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "&lt;ipython-input-2-052e1fda1dc0&gt;", line 1, in &lt;module&gt;
    runfile('/home/fatema/Downloads/practice1.py', wdir='/home/fatema/Downloads')
  File "/home/fatema/Downloads/pycharm-professional-2019.1.3/pycharm-2019.1.3/helpers/pydev/_pydev_bundle/pydev_umd.py", line 197, in runfile
    pydev_imports.execfile(filename, global_vars, local_vars)  # execute the script
  File "/home/fatema/Downloads/pycharm-professional-2019.1.3/pycharm-2019.1.3/helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile
    exec(compile(contents+"\n", file, 'exec'), glob, loc)
  File "/home/fatema/Downloads/practice1.py", line 100, in &lt;module&gt;
    sample = train_data[i]
  File "/home/fatema/Downloads/practice1.py", line 63, in __getitem__
    img_name = os.path.join(self.root_dir, str(self.data['ID'][idx])+'.npy')
  File "/home/fatema/miniconda3/lib/python3.6/site-packages/pandas/core/series.py", line 868, in __getitem__
    result = self.index.get_value(self, key)
  File "/home/fatema/miniconda3/lib/python3.6/site-packages/pandas/core/indexes/base.py", line 4375, in get_value
    tz=getattr(series.dtype, 'tz', None))
  File "pandas/_libs/index.pyx", line 81, in pandas._libs.index.IndexEngine.get_value
  File "pandas/_libs/index.pyx", line 89, in pandas._libs.index.IndexEngine.get_value
  File "pandas/_libs/index.pyx", line 132, in pandas._libs.index.IndexEngine.get_loc
  File "pandas/_libs/hashtable_class_helper.pxi", line 987, in pandas._libs.hashtable.Int64HashTable.get_item
  File "pandas/_libs/hashtable_class_helper.pxi", line 993, in pandas._libs.hashtable.Int64HashTable.get_item
KeyError: 0

</code></pre>
<p>sorry for asking again because I an very new in pytorch.</p>

Bibhabasu_Mohapatra: <p>Same is happening with me … <a class="mention" href="/u/mchowdhu">@mchowdhu</a>   if you have solved please help!!!</p>

mchowdhu: <p>This problem was solved by modifying the line of code  <code>img_name = os.path.join(self.root_dir, str(self.data['ID'].iloc[idx])+'.npy')</code> from <code>  img_name = os.path.join(self.root_dir, str(self.data['ID'][idx])+'.npy')</code> in the <code>  def __getitem__(self,idx):</code></p>

Bibhabasu_Mohapatra: <p>Thanks…   This solution removed that error…<br>
Did not expect that someone who had  2 years back<br>
Will see my message…</p>
<p>Thanks <a class="mention" href="/u/mchowdhu">@mchowdhu</a></p>

mihaelanistor5: <p>Hi everyone!</p>
<p>I have changed the def <strong>getitem</strong>(self,idx) function based on the solution from this post and I have encountered another issue that I do not know how to solve. Any input on how to fix it is much appreciated!<br>
<div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/3X/d/1/d107da920120cde34e9508d2ba9c2d6a226afdb1.png" data-download-href="https://discuss.pytorch.org/uploads/default/d107da920120cde34e9508d2ba9c2d6a226afdb1" title="image"><img src="https://discuss.pytorch.org/uploads/default/optimized/3X/d/1/d107da920120cde34e9508d2ba9c2d6a226afdb1_2_690x216.png" alt="image" data-base62-sha1="tPaCjyOx0Rg97sEq16MSJ2V7okx" width="690" height="216" srcset="https://discuss.pytorch.org/uploads/default/optimized/3X/d/1/d107da920120cde34e9508d2ba9c2d6a226afdb1_2_690x216.png, https://discuss.pytorch.org/uploads/default/optimized/3X/d/1/d107da920120cde34e9508d2ba9c2d6a226afdb1_2_1035x324.png 1.5x, https://discuss.pytorch.org/uploads/default/original/3X/d/1/d107da920120cde34e9508d2ba9c2d6a226afdb1.png 2x" data-dominant-color="EFEFEF"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">image</span><span class="informations">1341×420 60.7 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div></p>