Understanding how torch.nn.Module works

rasbt: Essentially, I want to reproduce the results I get when I do it “manually:” <SCODE>from torch.autograd import Variable
import torch


x = Variable(torch.Tensor([[1.0, 1.0], 
                           [1.0, 2.1], 
                           [1.0, 3.6], 
                           [1.0, 4.2], 
                           [1.0, 6.0], 
                           [1.0, 7.0]]))
y = Variable(torch.Tensor([1.0, 2.1, 3.6, 4.2, 6.0, 7.0]))
weights = Variable(torch.zeros(2, 1), requires_grad=True)


for i in range(5000):

    net_input = x.mm(weights)
    loss = torch.mean((net_input - y)**2)
    loss.backward()
    weights.data.add_(-0.0001 * weights.grad.data)
    
    if loss.data[0] < 1e-3:
        break

print('n_iter', i)
print(loss.data[0])
<ECODE> Output: <SCODE>n_iter 1188
0.0004487129335757345
<ECODE> Now, running the following <SCODE>import torch.nn.functional as F

class Model(torch.nn.Module):
    
    def __init__(self):
        super(Model, self).__init__()
        self.weights = Variable(torch.zeros(2, 1), 
                                requires_grad=True)
    
    def forward(self, x):
        net_input = x.mm(self.weights)
        return net_input
        
model = Model()
criterion = torch.nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.001)
<ECODE> raises an error: <SCODE>    ---------------------------------------------------------------------------
    IndexError                                Traceback (most recent call last)
    <ipython-input-258-3bcb3a8408d2> in <module>()
         15 model = Model()
         16 criterion = torch.nn.MSELoss()
    ---> 17 optimizer = torch.optim.SGD(model.parameters(), lr=0.001)

    /Users/Sebastian/miniconda3/envs/pytorch/lib/python3.5/site-packages/torch/optim/sgd.py in __init__(self, params, lr, momentum, dampening, weight_decay)
         24         defaults = dict(lr=lr, momentum=momentum, dampening=dampening,
         25                         weight_decay=weight_decay)
    ---> 26         super(SGD, self).__init__(params, defaults)
         27 
         28     def step(self, closure=None):

    /Users/Sebastian/miniconda3/envs/pytorch/lib/python3.5/site-packages/torch/optim/optimizer.py in __init__(self, params, defaults)
         25         self.state = defaultdict(dict)
         26         self.param_groups = list(params)
    ---> 27         if not isinstance(self.param_groups[0], dict):
         28             self.param_groups = [{'params': self.param_groups}]
         29 

    IndexError: list index out of range
<ECODE> I tried that: <SCODE>class Model(torch.nn.Module):
    
    def __init__(self):
        super(Model, self).__init__()
        self.weights = Variable(torch.zeros(2, 1), 
                        requires_grad=True)
        self.fc = torch.nn.Linear(2, 1)
        
    def forward(self, x):
        return x
        
model = Model()
criterion = torch.nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.0001)

for i in range(5000):
    optimizer.zero_grad()
    outputs = model(x)
    
    loss = criterion(outputs, y)
    loss.backward()        

    optimizer.step()
    
print(loss.data[0])
<ECODE> but now, I am getting an error about the dimensions of the input: <SCODE>    ---------------------------------------------------------------------------
    RuntimeError                              Traceback (most recent call last)
    <ipython-input-259-c6bb483f3953> in <module>()
         28     outputs = model(x)
         29 
    ---> 30     loss = criterion(outputs, y)
         31     loss.backward()
         32 

    /Users/Sebastian/miniconda3/envs/pytorch/lib/python3.5/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
        208 
        209     def __call__(self, *input, **kwargs):
    --> 210         result = self.forward(*input, **kwargs)
        211         for hook in self._forward_hooks.values():
        212             hook_result = hook(self, input, result)

    /Users/Sebastian/miniconda3/envs/pytorch/lib/python3.5/site-packages/torch/nn/modules/loss.py in forward(self, input, target)
         21         _assert_no_grad(target)
         22         backend_fn = getattr(self._backend, type(self).__name__)
    ---> 23         return backend_fn(self.size_average)(input, target)
         24 
         25 

    /Users/Sebastian/miniconda3/envs/pytorch/lib/python3.5/site-packages/torch/nn/_functions/thnn/auto.py in forward(self, input, target)
         39         output = input.new(1)
         40         getattr(self._backend, update_output.name)(self._backend.library_state, input, target,
    ---> 41             output, *self.additional_args)
         42         return output
         43 

    RuntimeError: input and target have different number of elements: input[6 x 2] has 12 elements, while target[6] has 6 elements at /Users/soumith/anaconda/conda-bld/pytorch-0.1.6_1484801351127/work/torch/lib/THNN/generic/MSECriterion.c:12
<ECODE>

apaszke: Both your examples have small errors inside: You have a bug in forward - your model always just returns the input. Try this: <SCODE>class Model(torch.nn.Module):
    
    def __init__(self):
        super(Model, self).__init__()
        self.fc = torch.nn.Linear(2, 1)
        
    def forward(self, x):
        return self.fc(x) # it was just x there
<ECODE>

rasbt: 

apaszke: Huh, I think we’re missing the docs on Parameters, I’ll note that and make sure it’s added soon. Sorry for the confusion.

tanemaki: Hi, thanks for clarifying the difference between Variable and Parameter in this thread. It might be slightly off the topic, but I have a question about the codes used in here. The very first code written by rasbt, which implemented OLS regression manually using only Variable, looks perfectly fine to me, and I got exactly same output. However, the loss is not monotonically decreasing for some unknown reason. Am I missing something? By the way, by replaced Variable used in the second code by rasbt to Parameter, the loss decreases monotonically as usual.

rasbt: Yeah, same issue here. The first example has some issues with the minimization: <SCODE>from torch.autograd import Variable
import torch


x = Variable(torch.Tensor([[1.0, 1.0], 
                           [1.0, 2.1], 
                           [1.0, 3.6], 
                           [1.0, 4.2], 
                           [1.0, 6.0], 
                           [1.0, 7.0]]))
y = Variable(torch.Tensor([1.0, 2.1, 3.6, 4.2, 6.0, 7.0]))
weights = Variable(torch.zeros(2, 1), requires_grad=True)

loss1 = []

for i in range(5000):

    net_input = x.mm(weights)
    loss = torch.mean((net_input - y)**2)
    loss.backward()
    weights.data.add_(-0.0001 * weights.grad.data)
    
    loss1.append(loss.data[0])

print('n_iter', i)
print(loss.data[0])

plt.plot(range(5000), loss1)
<ECODE> The 2nd example works well, though: <SCODE>import torch.nn.functional as F
from torch.nn import Parameter

class Model(torch.nn.Module):
    
    def __init__(self):
        super(Model, self).__init__()
        self.weights = Parameter(torch.zeros(2, 1), 
                                 requires_grad=True)
    
    def forward(self, x):
        net_input = x.mm(self.weights)
        return net_input
        
model = Model()
criterion = torch.nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.001)
loss2 = []

for i in range(5000):
    optimizer.zero_grad()
    outputs = model(x)
    
    loss = criterion(outputs, y)
    loss2.append(loss.data[0])
    loss.backward()        

    optimizer.step()
    
plt.plot(range(5000), loss2)
<ECODE> After standardizing ex1, and lowering the learning rate further, it would somewhat work though: <SCODE>from torch.autograd import Variable
import torch


x = torch.Tensor([[1.0, 1.0], 
                  [1.0, 2.1], 
                  [1.0, 3.6], 
                  [1.0, 4.2], 
                  [1.0, 6.0], 
                  [1.0, 7.0]])

x = (x - x.mean()) / x.max()
x = Variable(x)

y = torch.Tensor([1.0, 2.1, 3.6, 4.2, 6.0, 7.0])
y = (y - y.mean()) / y.max()
y = Variable(y)

weights = Variable(torch.zeros(2, 1), requires_grad=True)

loss1 = []

for i in range(5000):

    net_input = x.mm(weights)
    loss = torch.mean((net_input - y)**2)
    loss.backward()
    weights.data.add_(-0.00000001 * weights.grad.data)
    
    loss1.append(loss.data[0])

print('n_iter', i)
print(loss.data[0])

plt.plot(range(5000), loss1)
<ECODE>

apaszke: 

rasbt: 

Kalamaya: Good questions - Ive been trying to grok it too!

tanemaki: 

Atcold: 

rasbt: Regarding the symmetry: Yeah, in this case zero weights wouldn’t matter like you said

Kalamaya: Hmm, well why do you have to zero them out? You are just computing new gradients, so shouldn’t the old ones just be over-written?

apaszke: No, they’re accumulated.

hmishfaq: 

apaszke: Yes, that’s what I meant.

Alex_Choy: 

Brando_Miranda: its hard to follow what ur question is cuz ur title is vague and question details is too long.

vfdev-5: Hi, I would like to come back to this example and understand why there is a difference between two loss functions: loss1 = torch.mean((y - y_pred)**2.0) 
loss2 = mse_loss(y_pred, y), where mse_loss = nn.MSELoss()
 Here is my complete code: <SCODE>x = Variable(torch.FloatTensor([[1.0, 1.0], 
                                 [1.0, 2.1], 
                                 [1.0, 3.6], 
                                 [1.0, 4.2], 
                                 [1.0, 6.0], 
                                 [1.0, 7.0]]))
y = Variable(torch.FloatTensor([1.0, 2.1, 3.6, 4.2, 6.0, 7.0]))

mse_loss = nn.MSELoss()
weights = Variable(torch.zeros((2, 1)).float(), requires_grad=True)

n = 500
loss1 = []
loss2 = []
weights_grads_history = np.zeros((n, 2))
weights_history = np.zeros((n, 2))
learning_rate = 0.0001

for i in range(n):

    y_pred = x.mm(weights)
    loss = torch.mean((y - y_pred)**2.0)
    loss_ = mse_loss(y_pred, y)    
    loss1.append(loss.data[0])
    loss2.append(loss_.data[0])
    # Compute gradients
    loss.backward()
    
    # Update parameters
    weights_grads_history[i, :] = weights.grad.data.numpy()[:, 0]
    weights_history[i, :] = weights.data.numpy()[:, 0]
    weights.data.sub_(weights.grad.data * learning_rate)

    # You need to clear the existing gradients though, else gradients will be accumulated to existing gradients         
    weights.grad.data.zero_()

    
print("n_iter", i)
print(loss1[-1], loss2[-1])
print("y_pred: ", y_pred)
print("Weights : ", weights)

plt.figure(figsize=(12, 4))
plt.subplot(131)
plt.plot(range(n), loss1, label='loss1')
plt.plot(range(n), loss2, label='loss2')
_ = plt.legend()
plt.subplot(132)
plt.plot(range(n), weights_grads_history[:, 0], label='W_grad_1')
plt.plot(range(n), weights_grads_history[:, 1], label='W_grad_2')
plt.legend()
plt.subplot(133)
plt.plot(range(n), weights_history[:, 0], label='W_1')
plt.plot(range(n), weights_history[:, 1], label='W_2')
_ = plt.legend()
<ECODE> Thanks

ptrblck: Hi, Your code should work when you change the following line: <SCODE>loss = torch.mean((y - y_pred.view(-1))**2.0)<ECODE>