List of nn.Module in a nn.Module

NgPDat: <SCODE>class testNet(nn.Module):
    def __init__(self, input_dim, hidden_dim, step=1):
        super(testNet, self).__init__()
        self.linear = nn.Linear(100, 100) #dummy module
        self.linear_combines1 = []
        self.linear_combines2 = []
        for i in range(step):
            self.linear_combines1.append(nn.Linear(input_dim, hidden_dim))
            self.linear_combines2.append(nn.Linear(hidden_dim, hidden_dim))

net = testNet(128, 256, 3)
print(net) #Won't print what is in the list
net.cuda() #Won't send the module in the list to gpu
<ECODE> What is the intended correct way to do this?

fmassa: Hi, <SCODE>import torch
import torch.nn as nn

class ListModule(nn.Module):
    def __init__(self, *args):
        super(ListModule, self).__init__()
        idx = 0
        for module in args:
            self.add_module(str(idx), module)
            idx += 1

    def __getitem__(self, idx):
        if idx < 0 or idx >= len(self._modules):
            raise IndexError('index {} is out of range'.format(idx))
        it = iter(self._modules.values())
        for i in range(idx):
            next(it)
        return next(it)

    def __iter__(self):
        return iter(self._modules.values())

    def __len__(self):
        return len(self._modules)

class testNet(nn.Module):
    def __init__(self, input_dim, hidden_dim, step=1):
        super(testNet, self).__init__()
        self.linear = nn.Linear(100, 100) #dummy module
        linear_combines1 = []
        linear_combines2 = []
        for i in range(step):
            linear_combines1.append(nn.Linear(input_dim, hidden_dim))
            linear_combines2.append(nn.Linear(hidden_dim, hidden_dim))
        self.linear_combines1 = ListModule(*linear_combines1)
        self.linear_combines2 = ListModule(*linear_combines2)

net = testNet(128, 256, 3)
print(net)
net.cuda()

print(net.linear_combines1[0])
print(len(net.linear_combines2))
for i in net.linear_combines1:
    print(i.weight.data.type())
<ECODE>

apaszke: <SCODE>class AttrProxy(object):
    """Translates index lookups into attribute lookups."""
    def __init__(self, module, prefix):
        self.module = module
        self.prefix = prefix

    def __getitem__(self, i):
        return getattr(self.module, self.prefix + str(i))


class testNet(nn.Module):
    def __init__(self, input_dim, hidden_dim, steps=1):
        super(testNet, self).__init__()
        self.steps = steps
        for i in range(steps):
            self.add_module('i2h_' + str(i), nn.Linear(input_dim, hidden_dim))
            self.add_module('h2h_' + str(i), nn.Linear(hidden_dim, hidden_dim))
        self.i2h = AttrProxy(self, 'i2h_')
        self.h2h = AttrProxy(self, 'h2h_')

    def forward(self, input, hidden):
        # here, use self.i2h[t] and self.h2h[t] to index 
        # input2hidden and hidden2hidden modules for each step,
        # or loop over them, like in the example below
        # (assuming first dim of input is sequence length)
        for inp, i2h, h2h in zip(input, self.i2h, self.h2h):
            hidden = F.tanh(i2h(input) + h2h(hidden))
        return hidden
<ECODE>

NgPDat: Thank you! Both ideas are great. I took some time to incorporate two ideas together. And here is my take on it: <SCODE>class ListModule(object):
    #Should work with all kind of module
    def __init__(self, module, prefix, *args):
        self.module = module
        self.prefix = prefix
        self.num_module = 0
        for new_module in args:
            self.append(new_module)

    def append(self, new_module):
        if not isinstance(new_module, nn.Module):
            raise ValueError('Not a Module')
        else:
            self.module.add_module(self.prefix + str(self.num_module), new_module)
            self.num_module += 1

    def __len__(self):
        return self.num_module

    def __getitem__(self, i):
        if i < 0 or i >= self.num_module:
            raise IndexError('Out of bound')
        return getattr(self.module, self.prefix + str(i))


class testNet(nn.Module):
    def __init__(self, input_dim, hidden_dim, steps=1):
        super(testNet, self).__init__()
        self.steps = steps
        self.i2h = ListModule(self, 'i2h_')
        self.h2h = ListModule(self, 'h2h_')
        for i in range(steps):
            self.i2h.append(nn.Linear(input_dim, hidden_dim))
            self.h2h.append(nn.Linear(hidden_dim, hidden_dim))

    def forward(self, input, hidden):
        for inp, i2h, h2h in zip(input, self.i2h, self.h2h):
            hidden = F.tanh(i2h(inp) + h2h(hidden))
        return hidden

net = testNet(128, 256, 3)
print(net)
net.cuda()
inp = Variable(torch.randn(3, 4, 128)).cuda()
init = Variable(torch.randn(4, 256)).cuda()
out = net(inp, init)
<ECODE>

Atcold: 

NgPDat: 

apaszke: Yup, just wanted to wait until I write the docs before posting here. They should be up today or tomorrow.

Atcold: 

apaszke: The docs aren’t merged yet. I’ve been working on bug fixes.

Brando_Miranda: <SCODE>    self.W = Variable(w_init, requires_grad=True)
    self.mod_list = torch.nn.ModuleList([self.W])<ECODE>

fmassa: 

Brando_Miranda: A linear “module” without a bias is consider a module when its simply just a Variable (matrix with some dimension). I don’t see the difference of that and my suggestion. Something like: <SCODE>torch.nn.Linear(D_in,D_out,bias=False)<ECODE>

fmassa: <SCODE>def linear(input):
    x = Variable(torch.rand(2, 2))
    return input
<ECODE> and <SCODE>def linear(input):
    x = Variable(torch.rand(2, 2))
    return torch.matmul(input, x)
<ECODE>

BestCookies: