Certain class attributes are not recognized in forward()

username: AfonsoSalgadoSousa
cooked: <p>Hello. I am experiencing a strange behaviour on the forward() method of nn.Module. I have a class that inherits nn.Module and initialize some attributes on the <strong>init</strong>() method. The forward() in this class does not recognize the attribute that is itself an nn.Module.</p>
<p>The context of this error is while trying to run an implementation for DSQ (paper: <a href="https://arxiv.org/pdf/1908.05033.pdf" rel="noopener nofollow ugc">https://arxiv.org/pdf/1908.05033.pdf</a>, code: <a href="https://github.com/ricky40403/DSQ" rel="noopener nofollow ugc">https://github.com/ricky40403/DSQ</a>). Either I get “torch.nn.modules.module.ModuleAttributeError: ‘DSQConv’ object has no attribute ‘running_lw’”, or in QuantConv I get “‘QuantMesure object has no attribute ‘quant’””.</p>

username: ptrblck
cooked: <p>Could you post a code snippet to reproduce this behavior, please?</p>

username: AfonsoSalgadoSousa
cooked: <p>Yes, of course. Thanks for the reply.</p>
<p>Using the code from a PyTorch implementation of DSQ (paper: <a href="https://arxiv.org/pdf/1908.05033.pdf" rel="noopener nofollow ugc">https://arxiv.org/pdf/1908.05033.pdf</a>, code: <a href="https://github.com/ricky40403/DSQ" class="inline-onebox" rel="noopener nofollow ugc">GitHub - ricky40403/DSQ: pytorch implementation of "Differentiable Soft Quantization: Bridging Full-Precision and Low-Bit Neural Networks"</a>), the error comes from this class (trimmed some additional functions):</p>
<pre><code class="lang-auto">class DSQConv(nn.Conv2d):
    def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, bias=True,
                momentum = 0.1,                
                num_bit = 8, QInput = True, bSetQ = True):
        super(DSQConv, self).__init__(in_channels, out_channels, kernel_size, stride, padding, dilation, groups, bias)
        self.num_bit = num_bit
        self.quan_input = QInput
        self.bit_range = 2**self.num_bit -1	 
        self.is_quan = bSetQ        
        self.momentum = momentum
        if self.is_quan:
            # using int32 max/min as init and backprogation to optimization
            # Weight
            self.uW = nn.Parameter(data = torch.tensor(2 **31 - 1).float())
            self.lW  = nn.Parameter(data = torch.tensor((-1) * (2**32)).float())
            self.register_buffer('running_uw', torch.tensor([self.uW.data])) # init with uw
            self.register_buffer('running_lw', torch.tensor([self.lW.data])) # init with lw
            self.alphaW = nn.Parameter(data = torch.tensor(0.2).float())
            # Bias
            if self.bias is not None:
                self.uB = nn.Parameter(data = torch.tensor(2 **31 - 1).float())
                self.lB  = nn.Parameter(data = torch.tensor((-1) * (2**32)).float())
                self.register_buffer('running_uB', torch.tensor([self.uB.data]))# init with ub
                self.register_buffer('running_lB', torch.tensor([self.lB.data]))# init with lb
                self.alphaB = nn.Parameter(data = torch.tensor(0.2).float())
              
            # Activation input		
            if self.quan_input:
                self.uA = nn.Parameter(data = torch.tensor(2 **31 - 1).float())
                self.lA  = nn.Parameter(data = torch.tensor((-1) * (2**32)).float())
                self.register_buffer('running_uA', torch.tensor([self.uA.data])) # init with uA
                self.register_buffer('running_lA', torch.tensor([self.lA.data])) # init with lA
                self.alphaA = nn.Parameter(data = torch.tensor(0.2).float())

    def forward(self, x):
        if self.is_quan:
            # Weight Part
            # moving average
            if self.training:
                cur_running_lw = self.running_lw.mul(1-self.momentum).add((self.momentum) * self.lW)
                cur_running_uw = self.running_uw.mul(1-self.momentum).add((self.momentum) * self.uW)
            else:
                cur_running_lw = self.running_lw
                cur_running_uw = self.running_uw

            Qweight = self.clipping(self.weight, cur_running_uw, cur_running_lw)
            cur_max = torch.max(Qweight)
            cur_min = torch.min(Qweight)
            delta =  (cur_max - cur_min)/(self.bit_range)
            interval = (Qweight - cur_min) //delta            
            mi = (interval + 0.5) * delta + cur_min
            Qweight = self.phi_function(Qweight, mi, self.alphaW, delta)
            Qweight = self.sgn(Qweight)
            Qweight = self.dequantize(Qweight, cur_min, delta, interval)

            Qbias = self.bias
            # Bias			
            if self.bias is not None:
                # self.running_lB.mul_(1-self.momentum).add_((self.momentum) * self.lB)
                # self.running_uB.mul_(1-self.momentum).add_((self.momentum) * self.uB)
                if self.training:
                    cur_running_lB = self.running_lB.mul(1-self.momentum).add((self.momentum) * self.lB)
                    cur_running_uB = self.running_uB.mul(1-self.momentum).add((self.momentum) * self.uB)
                else:
                    cur_running_lB = self.running_lB
                    cur_running_uB = self.running_uB

                Qbias = self.clipping(self.bias, cur_running_uB, cur_running_lB)
                cur_max = torch.max(Qbias)
                cur_min = torch.min(Qbias)
                delta =  (cur_max - cur_min)/(self.bit_range)
                interval = (Qbias - cur_min) //delta
                mi = (interval + 0.5) * delta + cur_min
                Qbias = self.phi_function(Qbias, mi, self.alphaB, delta)
                Qbias = self.sgn(Qbias)
                Qbias = self.dequantize(Qbias, cur_min, delta, interval)

            # Input(Activation)
            Qactivation = x
            if self.quan_input:
                                
                if self.training:                    
                    cur_running_lA = self.running_lA.mul(1-self.momentum).add((self.momentum) * self.lA)
                    cur_running_uA = self.running_uA.mul(1-self.momentum).add((self.momentum) * self.uA)
                else:
                    cur_running_lA = self.running_lA
                    cur_running_uA = self.running_uA
                    
                Qactivation = self.clipping(x, cur_running_uA, cur_running_lA)
                cur_max = torch.max(Qactivation)
                cur_min = torch.min(Qactivation)
                delta =  (cur_max - cur_min)/(self.bit_range)
                interval = (Qactivation - cur_min) //delta
                mi = (interval + 0.5) * delta + cur_min                
                Qactivation = self.phi_function(Qactivation, mi, self.alphaA, delta)
                Qactivation = self.sgn(Qactivation)
                Qactivation = self.dequantize(Qactivation, cur_min, delta, interval)
            
            output = F.conv2d(Qactivation, Qweight, Qbias,  self.stride, self.padding, self.dilation, self.groups)

        else:
            output =  F.conv2d(x, self.weight, self.bias, self.stride,
                        self.padding, self.dilation, self.groups)

        return output
</code></pre>
<p>I get the following error message:</p>
<blockquote>
<p>Traceback (most recent call last):<br>
File “train.py”, line 579, in <br>
main()<br>
File “train.py”, line 137, in main<br>
main_worker(args.gpu, ngpus_per_node, args)<br>
File “train.py”, line 320, in main_worker<br>
train(train_loader, model, criterion, optimizer, epoch, args)<br>
File “train.py”, line 407, in train<br>
output = model(images)<br>
File “/home/afonso/anaconda3/envs/nlp/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 722, in _call_impl<br>
result = self.forward(*input, **kwargs)<br>
File “/home/afonso/anaconda3/envs/nlp/lib/python3.7/site-packages/torch/nn/parallel/data_parallel.py”, line 153, in forward<br>
return self.module(*inputs[0], **kwargs[0])<br>
File “/home/afonso/anaconda3/envs/nlp/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 722, in _call_impl<br>
result = self.forward(*input, **kwargs)<br>
File “/home/afonso/anaconda3/envs/nlp/lib/python3.7/site-packages/torchvision/models/resnet.py”, line 220, in forward<br>
return self._forward_impl(x)<br>
File “/home/afonso/anaconda3/envs/nlp/lib/python3.7/site-packages/torchvision/models/resnet.py”, line 203, in _forward_impl<br>
x = self.conv1(x)<br>
File “/home/afonso/anaconda3/envs/nlp/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 722, in _call_impl<br>
result = self.forward(*input, **kwargs)<br>
File “/home/afonso/Projects/quantization/DSQ/DSQConv.py”, line 88, in forward<br>
cur_running_lw = self.running_lw.mul(1-self.momentum).add((self.momentum) * self.lW)<br>
File “/home/afonso/anaconda3/envs/nlp/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 772, in <strong>getattr</strong><br>
type(self).<strong>name</strong>, name))<br>
torch.nn.modules.module.ModuleAttributeError: ‘DSQConv’ object has no attribute ‘running_lw’</p>
</blockquote>
<p>Additionally, from my implementation of a post-training uniform quantization for 3D models, adapted with some classes from that same DSQ project, for the following code:</p>
<pre><code class="lang-auto">class QuantConv3d(nn.Conv3d):
    def __init__(self, in_channels, out_channels, kernel_size,
                 stride=1, padding=0, dilation=1, groups=1, bias=True, num_bits=8, num_bits_weight=None, momentum=0.1):
        super(QuantConv3d, self).__init__(in_channels, out_channels, kernel_size,
                                          stride, padding, dilation, groups, bias)
        self.num_bits = num_bits
        self.num_bits_weight = num_bits_weight or num_bits
        self.quantize_input = QuantMeasure(num_bits=num_bits, momentum=momentum)

    def forward(self, input):
        input = self.quantize_input(input)
        qweight = quantize(self.weight, num_bits=self.num_bits_weight,
                           min_value=float(self.weight.min()),
                           max_value=float(self.weight.max()))
        if self.bias is not None:
            qbias = quantize(self.bias, num_bits=self.num_bits_weight)
        else:
            qbias = None

        output = F.conv3d(input, qweight, qbias, self.stride,
                          self.padding, self.dilation, self.groups)
        return output
</code></pre>
<p>I get the following error message:</p>
<blockquote>
<p>Traceback (most recent call last):<br>
File “main.py”, line 155, in <br>
main()<br>
File “main.py”, line 97, in main<br>
opt.device)<br>
File “/home/ctm/afonso/easyride/acceleration/src/core/trainer.py”, line 41, in train_epoch<br>
outputs = model(inputs)<br>
File “/home/ctm/.conda/envs/3dcnn/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 722, in _call_impl<br>
result = self.forward(*input, **kwargs)<br>
File “/home/ctm/.conda/envs/3dcnn/lib/python3.7/site-packages/torch/nn/parallel/data_parallel.py”, line 153, in forward<br>
return self.module(*inputs[0], **kwargs[0])<br>
File “/home/ctm/.conda/envs/3dcnn/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 722, in _call_impl<br>
result = self.forward(*input, **kwargs)<br>
File “/home/ctm/afonso/easyride/acceleration/src/models/mobilenetv2.py”, line 126, in forward<br>
x = self.features(x)<br>
File “/home/ctm/.conda/envs/3dcnn/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 722, in _call_impl<br>
result = self.forward(*input, **kwargs)<br>
File “/home/ctm/.conda/envs/3dcnn/lib/python3.7/site-packages/torch/nn/modules/container.py”, line 117, in forward<br>
input = module(input)<br>
File “/home/ctm/.conda/envs/3dcnn/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 722, in _call_impl<br>
result = self.forward(*input, **kwargs)<br>
File “/home/ctm/.conda/envs/3dcnn/lib/python3.7/site-packages/torch/nn/modules/container.py”, line 117, in forward<br>
input = module(input)<br>
File “/home/ctm/.conda/envs/3dcnn/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 722, in _call_impl<br>
result = self.forward(*input, **kwargs)<br>
File “/home/ctm/afonso/easyride/acceleration/src/quantization/uni_quant_3d.py”, line 139, in forward<br>
input = self.quantize_input(input)<br>
File “/home/ctm/.conda/envs/3dcnn/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 772, in <strong>getattr</strong><br>
type(self).<strong>name</strong>, name))<br>
torch.nn.modules.module.ModuleAttributeError: ‘QuantConv3d’ object has no attribute ‘quantize_input’</p>
</blockquote>
<p>Both implementations use the PyTransformer (<a href="https://github.com/ricky40403/PyTransformer/blob/master/transformers/torchTransformer.py" rel="noopener nofollow ugc">https://github.com/ricky40403/PyTransformer/blob/master/transformers/torchTransformer.py</a>) trans_layer method to permute standard conv layers to quantized ones. The code goes as follows:</p>
<pre><code class="lang-auto">def trans_layers(self, model, update = True):
		"""!
		This function transform layer by layers in register dictionaries
		@param model: input model to transfer
		@param update: default is True, whether to update the parameter from the original layer or not. 
		Note that it will update matched parameters only.
		@return transformed model
		"""
		# print("trans layer")
		if len(self._register_dict) == 0:
			print("No layer to swap")
			print("Please use register( {origin_layer}, {target_layer} ) to register layer")
			return model
		else:
			for module_name in model._modules:			
				# has children
				if len(model._modules[module_name]._modules) &gt; 0:
					self.trans_layers(model._modules[module_name])
				else:
					if type(getattr(model, module_name)) in self._register_dict:
						# use inspect.signature to know args and kwargs of __init__
						_sig = inspect.signature(type(getattr(model, module_name)))
						_kwargs = {}
						for key in _sig.parameters:
							if _sig.parameters[key].default == inspect.Parameter.empty: #args 
								# assign args
								# default values should be handled more properly, unknown data type might be an issue
								if 'kernel' in key:
									# _sig.parameters[key].replace(default=inspect.Parameter.empty, annotation=3)
									value = 3
								elif 'channel' in key:
									# _sig.parameters[key].replace(default=inspect.Parameter.empty, annotation=32)
									value = 32
								else:
									# _sig.parameters[key].replace(default=inspect.Parameter.empty, annotation=None)
									value = None
						
								_kwargs[key] = value

						_attr_dict = getattr(model, module_name).__dict__
						_layer_new = self._register_dict[type(getattr(model, module_name))](**_kwargs) # only give positional args
						_layer_new.__dict__.update(_attr_dict)

						setattr(model, module_name, _layer_new)
		return model
</code></pre>
<p>Thank you for your help in advance.</p>