How to use OUSMLoss?

mobassir94: <p>i am trying to use OSUMLoss like this :</p>
<pre><code class="lang-auto">
def get_loss(name):
    loss_dict = {
        'CrossEntropy': F.cross_entropy,
        'SoftCrossEntropy': soft_cross_entropy_loss,
        'LabelSmoothingCrossEntropy':
        LabelSmoothingCrossEntropy(epsilon=0.1),
        'SCE': SymmetricCrossEntropy(), 
        'BCEWithLogitsLoss': F.binary_cross_entropy_with_logits,
        'Coral': coral_loss, 
        'MSELoss': F.mse_loss,
        'MAELoss': F.l1_loss,
        'huber': F.smooth_l1_loss,
    }
    return loss_dict[name]

class OUSMLoss(nn.Module):
    '''
    Implementation of 
    Loss with Online Uncertainty Sample Mining:
    https://arxiv.org/pdf/1901.07759.pdf
    # Params
    k: num of samples to drop in a mini batch
    loss: loss function name (see get_loss function above)
    trigger: the epoch it starts to train on OUSM (please call `.update(epoch)` each epoch)
    '''

    def __init__(self, k=1,  trigger=5, ousm=True):
        super(OUSMLoss, self).__init__()
        self.k = k
        self.loss_name = 'CrossEntropy'
        self.loss = get_loss(self.loss_name)
        self.trigger = trigger
        self.ousm = ousm

    def forward(self, logits, targets, indices=None):
        logits, targets = _check_input_type(logits, targets, self.loss_name)
        bs = logits.shape[0]
        if self.ousm and bs - self.k &gt; 0:
            losses = self.loss(logits, targets, reduction='none')
            if len(losses.shape) == 2:
                losses = losses.mean(1)
            _, idxs = losses.topk(bs-self.k, largest=False)
            losses = losses.index_select(0, idxs)
            return losses.mean()
        else:
            return self.loss(logits, targets)

    def update(self, current_epoch):
        self.current_epoch = current_epoch
        if current_epoch == self.trigger:
            self.ousm = True
            print('criterion: ousm is True.')

    def __repr__(self):
        return f'OUSM(loss={self.loss_name}, k={self.k}, trigger={self.trigger}, ousm={self.ousm})'
</code></pre>
<p>ref :  <a href="https://github.com/analokmaus/kaggle-panda-challenge-public/blob/208caf4c83a5ab9d181e66eee447cd2e475d58dc/models/noisy_loss.py" class="inline-onebox" rel="noopener nofollow ugc">kaggle-panda-challenge-public/noisy_loss.py at 208caf4c83a5ab9d181e66eee447cd2e475d58dc · analokmaus/kaggle-panda-challenge-public · GitHub</a></p>
<p>when i do  :</p>
<pre><code class="lang-auto">loss = OUSMLoss(logits, target)
loss.backward()
</code></pre>
<p>i get,</p>
<pre><code class="lang-auto">---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
&lt;timed exec&gt; in &lt;module&gt;

&lt;ipython-input-105-6eecc42fef33&gt; in train_epoch(loader, optimizer)
     16         loss = OUSMLoss(logits, target)
     17 
---&gt; 18         loss.backward()
     19         optimizer.step()
     20 

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py in __getattr__(self, name)
    592                 return modules[name]
    593         raise AttributeError("'{}' object has no attribute '{}'".format(
--&gt; 594             type(self).__name__, name))
    595 
    596     def __setattr__(self, name, value):

AttributeError: 'OUSMLoss' object has no attribute 'backward'
</code></pre>
<p>how can i solve this issue? <a class="mention" href="/u/ptrblck">@ptrblck</a></p>

ptrblck: <p><code>OUSMLoss</code> is defined as an <code>nn.Module</code>, while <code>.backward()</code> is a <code>tensor</code> method.<br>
You would either have to implement the <code>backward()</code> method in this module or call <code>.backward()</code> on the <code>loss</code> tensor (probably the return tensor).</p>

mobassir94: <p>dear <a class="mention" href="/u/ptrblck">@ptrblck</a></p>
<p>i was trying to use  OUSMLoss from this repo :  <a href="https://github.com/analokmaus/kaggle-panda-challenge-public/blob/master/models/noisy_loss.py" class="inline-onebox" rel="noopener nofollow ugc">kaggle-panda-challenge-public/noisy_loss.py at master · analokmaus/kaggle-panda-challenge-public · GitHub</a></p>
<p>for a multi class (5 class) image classification problem.</p>
<p>i was trying to use OUSMLoss like this :</p>
<pre><code class="lang-auto">class myOUSMLoss(nn.Module):
    '''
    Implementation of 
    Loss with Online Uncertainty Sample Mining:
    https://arxiv.org/pdf/1901.07759.pdf
    # Params
    k: num of samples to drop in a mini batch
    loss: loss function name (see get_loss function above)
    trigger: the epoch it starts to train on OUSM (please call `.update(epoch)` each epoch)
    '''

    def __init__(self, k=1,  trigger=0, ousm=True):
        super(myOUSMLoss, self).__init__()
        self.k = k
  
        
        self.trigger = trigger
        self.ousm = ousm

    def forward(self, logits, targets, indices=None):
        bs = logits.shape[0]
        print(bs)
        print(bs - self.k)
        if self.ousm and bs - self.k &gt; 0:
            #losses = self.loss(logits, targets, reduction='none')
            losses = bi_tempered_logistic_loss(logits, targets, t1=t1, t2=t2, label_smoothing=smoothing)
            if len(losses.shape) == 2:
                losses = losses.mean(1)
            _, idxs = losses.topk(bs-self.k, largest=False)
            losses = losses.index_select(0, idxs)
            return losses.mean()

</code></pre>
<p><strong>and then from train one epoch and valid one epoch function just using loss function like this :</strong></p>
<pre><code class="lang-auto">myloss = myOUSMLoss()
loss = myloss(data, target)

</code></pre>
<p>and it gives me the following error :</p>
<pre><code class="lang-auto">
16
14
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
&lt;ipython-input-18-d97376da71cb&gt; in &lt;module&gt;()
----&gt; 1 get_ipython().run_cell_magic('time', '', '\nacc_max = 0.\nbest_file = f\'/content/drive/MyDrive/kaggledatasets/cassavamodels/{kernel_type}_best_fold{fold}.pth\'\nfor epoch in range(1, n_epochs+1):\n    print(time.ctime(), \'Epoch:\', epoch)\n    scheduler.step(epoch-1)\n\n    train_loss = train_epoch(train_loader, optimizer)\n    val_loss, acc = val_epoch(valid_loader)\n\n    content = time.ctime() + \' \' + f\'Epoch {epoch}, lr: {optimizer.param_groups[0]["lr"]:.7f}, train loss: {np.mean(train_loss):.5f}, val loss: {np.mean(val_loss):.5f}, acc: {(acc):.5f}\'\n    print(content)\n    with open(f\'/content/drive/MyDrive/kaggledatasets/cassavamodels/log_{kernel_type}.txt\', \'a\') as appender:\n        appender.write(content + \'\\n\')\n\n    if acc &gt; acc_max:\n        print(\'acc ({:.6f} --&gt; {:.6f}).  Saving model ...\'.format(acc_max, acc))\n        torch.save(model.state_dict(), best_file)\n        acc_max = acc\n\n\n#torch.save(model.state_dict(), os.path.join(f\'/content/drive/MyDrive/kaggledatasets/cassavamodels/{kernel_type}_final_fold{fold}.pth\'))')
5 frames
&lt;decorator-gen-60&gt; in time(self, line, cell, local_ns)
&lt;timed exec&gt; in &lt;module&gt;()
&lt;ipython-input-14-5055b70e47f6&gt;
 in forward(self, logits, targets, indices)
     31             if len(losses.shape) == 2:
     32                 losses = losses.mean(1)
---&gt; 33             _, idxs = losses.topk(bs-self.k, largest=False)
     34             losses = losses.index_select(0, idxs)
     35             return losses.mean()
RuntimeError: invalid argument 5: k not in range for dimension at /pytorch/aten/src/THC/generic/THCTensorTopK.cu:26
</code></pre>
<p>how can i fix this issue?</p>

ptrblck: <p>This error might be raised, if you are trying to select the top <code>k</code> values, while the size of the specified dimension is smaller than <code>k</code>.<br>
Check the shape of <code>losses</code> and make sure it contains more than <code>bs-self.k</code> values.</p>

mobassir94: <p><a class="mention" href="/u/ptrblck">@ptrblck</a>  i checked that bs - self.k is  <strong>15</strong></p>
<p>but when i tried :</p>
<p><code>print("losses.shape ",losses.shape)</code></p>
<p>and</p>
<p><code>print("losses ",list(losses.size()))</code></p>
<p>i get this output :</p>
<pre><code class="lang-auto">losses.shape  torch.Size([])

losses  []
</code></pre>
<p>what am i missing?</p>

mobassir94: <p><a class="mention" href="/u/ptrblck">@ptrblck</a>  fixed that issue and now getting Nan</p>
<p>this is the code i am using :</p>
<pre><code class="lang-auto">def soft_cross_entropy_loss(logits, targets, weights=1, reduction='none'):

    if len(targets.shape) == 1 or targets.shape[1] == 1:

        onehot_targets = torch.eye(logits.shape[1])[targets].to(logits.device)

    else:

        onehot_targets = targets

    loss = -torch.sum(onehot_targets * F.log_softmax(logits, 1), 1)

    if reduction == 'none':

        return loss

    elif reduction == 'sum':

        return loss.sum()

    elif reduction == 'mean':

        return loss.mean()
</code></pre>
<pre><code class="lang-auto">def ousm(logits, targets, indices=None):

    logits, targets = _check_input_type(logits, targets, 'SoftCrossEntropy')

    bs = logits.shape[0]

    k = 1

    print(bs)

    print("bs - k ", bs - k)

    

    if bs - k &gt; 0:

      #losses = self.loss(logits, targets)

      losses = soft_cross_entropy_loss(logits, targets, reduction='none')

      #losses = bi_tempered_logistic_loss(logits, targets, t1=t1, t2=t2, label_smoothing=smoothing)

      print(losses)

      print("losses.shape ",losses.shape)

      print("losses ",list(losses.size()))

      print(len(losses.shape))

      if len(losses.shape) == 2:

          losses = losses.mean(1)

      print("losses.shape test ",losses.shape)

      print("losses test",list(losses.size()))

      _, idxs = losses.topk(bs-k, largest=True)

      losses = losses.index_select(0, idxs)

      return losses.mean()
</code></pre>
<p>and i get output like this :</p>
<pre><code class="lang-auto">
16
bs - k  15
tensor([1.5355, 1.6762, 1.6320, 1.5614, 1.5086, 1.6454, 1.4942, 1.5916, 1.6462,
        1.7642, 1.4607, 1.5639, 1.6708, 1.4343, 1.7360, 1.6173],
       device='cuda:0', grad_fn=&lt;NegBackward&gt;)
losses.shape  torch.Size([16])
losses  [16]
1
losses.shape test  torch.Size([16])
losses test [16]
</code></pre>
<p>after some times,</p>
<pre><code class="lang-auto">16
bs - k  15
tensor([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan],
       device='cuda:0', grad_fn=&lt;NegBackward&gt;)
losses.shape  torch.Size([16])
losses  [16]
1
losses.shape test  torch.Size([16])
losses test [16]
</code></pre>

ptrblck: <p>I would recommend to check the custom loss function and make sure no invalid operations are used, such as a zero division etc.<br>
Anomaly detection might also be helpful to further isolate the issue.</p>