Why loss is always equal to 0.69

learner11: <p>“”" This module creates SRNet model.“”"<br>
import torch<br>
from torch import Tensor<br>
from torch import nn<br>
from model.utils import Type1, Type2, Type3, Type4</p>
<p>class Srnet(nn.Module):<br>
“”“This is SRNet model class.”“”</p>
<pre><code>def __init__(self) -&gt; None:
    """Constructor."""
    super().__init__()
    self.type1s = nn.Sequential(Type1(1, 64), Type1(64, 16))
    self.type2s = nn.Sequential(
        Type2(16, 16),
        Type2(16, 16),
        Type2(16, 16),
        Type2(16, 16),
        Type2(16, 16),
    )
    self.type3s = nn.Sequential(
        Type3(16, 16),
        Type3(16, 64),
        Type3(64, 128),
        Type3(128, 256),
    )
    self.type4 = Type4(256, 512)
    self.dense = nn.Linear(512, 2)
    self.softmax = nn.LogSoftmax(dim=1)

def forward(self, inp: Tensor) -&gt; Tensor:
    """Returns logits for input images.
    Args:
        inp (Tensor): input image tensor of shape (Batch, 1, 256, 256)
    Returns:
        Tensor: Logits of shape (Batch, 2)
    """
    out = self.type1s(inp)
    out = self.type2s(out)
    out = self.type3s(out)
    out = self.type4(out)
    out = out.view(out.size(0), -1)
    out = self.dense(out)
    return self.softmax(out)
</code></pre>
<p>class ConvBn(nn.Module):<br>
“”“Provides utility to create different types of layers.”“”</p>
<pre><code>def __init__(self, in_channels: int, out_channels: int) -&gt; None:
    """Constructor.
    Args:
        in_channels (int): no. of input channels.
        out_channels (int): no. of output channels.
    """
    super().__init__()
    self.conv = nn.Conv2d(
        in_channels,
        out_channels,
        kernel_size=3,
        stride=1,
        padding=1,
        bias=True,
    )
    self.batch_norm = nn.BatchNorm2d(out_channels)

def forward(self, inp: Tensor) -&gt; Tensor:
    """Returns Conv2d followed by BatchNorm.

    Returns:
        Tensor: Output of Conv2D -&gt; BN.
    """
    return self.batch_norm(self.conv(inp))
</code></pre>
<p>class Type1(nn.Module):<br>
“”“Creates type 1 layer of SRNet.”“”</p>
<pre><code>def __init__(self, in_channels: int, out_channels: int) -&gt; None:
    super().__init__()
    self.convbn = ConvBn(in_channels, out_channels)
    self.relu = nn.ReLU()

def forward(self, inp: Tensor) -&gt; Tensor:
    """Returns type 1 layer of SRNet.
    Args:
        inp (Tensor): input tensor.
    Returns:
        Tensor: Output of type 1 layer.
    """
    return self.relu(self.convbn(inp))
</code></pre>
<p>class Type2(nn.Module):<br>
“”“Creates type 2 layer of SRNet.”“”</p>
<pre><code>def __init__(self, in_channels: int, out_channels: int) -&gt; None:
    super().__init__()
    self.type1 = Type1(in_channels, out_channels)
    self.convbn = ConvBn(in_channels, out_channels)

def forward(self, inp: Tensor) -&gt; Tensor:
    """Returns type 2 layer of SRNet.
    Args:
        inp (Tensor): input tensor.
    Returns:
        Tensor: Output of type 2 layer.
    """
    return inp + self.convbn(self.type1(inp))
</code></pre>
<p>class Type3(nn.Module):<br>
“”“Creates type 3 layer of SRNet.”“”</p>
<pre><code>def __init__(self, in_channels: int, out_channels: int) -&gt; None:
    super().__init__()
    self.conv1 = nn.Conv2d(
        in_channels,
        out_channels,
        kernel_size=1,
        stride=2,
        padding=0,
        bias=True,
    )
    self.batch_norm = nn.BatchNorm2d(out_channels)
    self.type1 = Type1(in_channels, out_channels)
    self.convbn = ConvBn(out_channels, out_channels)
    self.pool = nn.AvgPool2d(kernel_size=3, stride=2, padding=1)

def forward(self, inp: Tensor) -&gt; Tensor:
    """Returns type 3 layer of SRNet.
    Args:
        inp (Tensor): input tensor.

    Returns:
        Tensor: Output of type 3 layer.
    """
    out = self.batch_norm(self.conv1(inp))
    out1 = self.pool(self.convbn(self.type1(inp)))
    return out + out1
</code></pre>
<p>class Type4(nn.Module):<br>
“”“Creates type 4 layer of SRNet.”“”</p>
<pre><code>def __init__(self, in_channels: int, out_channels: int) -&gt; None:
    super().__init__()
    self.type1 = Type1(in_channels, out_channels)
    self.convbn = ConvBn(out_channels, out_channels)
    self.gap = nn.AdaptiveAvgPool2d(output_size=1)

def forward(self, inp: Tensor) -&gt; Tensor:
    """Returns type 4 layer of SRNet.
    Args:
        inp (Tensor): input tensor.
    Returns:
        Tensor: Output of type 4 layer.
    """
    return self.gap(self.convbn(self.type1(inp)))
</code></pre>
<p>“”“This module provides utility function for training.”“”<br>
import os<br>
import re<br>
from typing import Any, Dict<br>
import torch<br>
from torch import nn</p>
<p>from opts.options import arguments</p>
<p>opt = arguments()</p>
<p>def saver(state: Dict[str, float], save_dir: str, epoch: int) → None:<br>
torch.save(state, save_dir + “net_” + str(epoch) + “.pt”)</p>
<p>def latest_checkpoint() → int:<br>
“”“Returns latest checkpoint.”“”<br>
if os.path.exists(opt.checkpoints_dir):<br>
all_chkpts = “”.join(os.listdir(opt.checkpoints_dir))<br>
if len(all_chkpts) &gt; 0:<br>
latest = max(map(int, re.findall(“\d+”, all_chkpts)))<br>
else:<br>
latest = None<br>
else:<br>
latest = None<br>
return latest</p>
<p>def adjust_learning_rate(optimizer: Any, epoch: int) → None:<br>
“”“Sets the learning rate to the initial learning_rate and decays by 10<br>
every 30 epochs.”“”<br>
learning_rate = opt.lr * (0.1 ** (epoch // 30))<br>
for param_group in optimizer.param_groups:<br>
param_group[“lr”] = learning_rate</p>
<h1><a name="weight-initialization-for-conv-layers-and-fc-layers-1" class="anchor" href="#weight-initialization-for-conv-layers-and-fc-layers-1"></a>Weight initialization for conv layers and fc layers</h1>
<p>def weights_init(param: Any) → None:<br>
“”“Initializes weights of Conv and fully connected.”“”</p>
<pre><code>if isinstance(param, nn.Conv2d):
    nn.init.kaiming_normal_(param.weight, mode='fan_out', nonlinearity='relu')
    if param.bias is not None:
        nn.init.constant_(param.bias, 0.2)
elif isinstance(param, nn.BatchNorm2d):
    nn.init.constant_(param.weight, 1)
    nn.init.constant_(param.bias, 0)
elif isinstance(param, nn.Linear):
    nn.init.normal_(param.weight, 0, 0.01)
    nn.init.constant_(param.bias, 0)
</code></pre>

ptrblck: <p>Your model doesn’t seem to train properly and stays at the initial loss value of <code>0.69</code>, which is expected for a 2-class multi-class classification use case (<code>-1.0 * log(class_prob) = -1.0 * log(0.5) = 0.69314</code>).<br>
Your code is unfortunately not properly formatted, but so far I didn’t see any obvious issues by skimming through it.</p>

learner11: <p>thanks,many people tell me that the problem may be about bn layer,the paper uses tensorflow 1.x,but when we use pytorch to do the same thing, it is wrong.Nobody can solve it.</p>

ptrblck: <p>Why should the problem point to the batchnorm layer?</p>

learner11: <p>i don’t know.But many people tell me the problem is about the difference of tensorflow and pytorch’s bn layer</p>

ptrblck: <p>I’m not aware of a difference between them but since many people tell you so you could ask one what the difference would be.</p>