How to traverse and copy a CNN pre-trained model by entering layer by layer?

gota_12: <p>Hi,<br>
I am trying to make a code that allows me to generalize the layers of a pre-trained model to generate a metamodel that is equivalent to twice the original model.</p>
<p>For this, I am using the convolutional layers that, using the group size concept, generate two outputs, and for now, I want the rest of the layers to be duplicated simply. In such a way, the first half of the convolution output will go to layer, for example, relu 1_1, and the second to relu1_2.</p>
<p>The main problem I have found is that many pre-trained networks are not defined using only simple layers but instead use groupings of layers in their topology.</p>
<p>In the following code:</p>
<pre><code class="lang-auto">    for name, layer in model.named_children():
        if (isinstance(layer, nn.Sequential) or list(layer.children())):
            for subname, sublayer in layer.named_children():
</code></pre>
<p>I have used it as a condition to try to differentiate between simple layers and groupings of layers, but even so, in most models, there are groupings of layers that the condition takes as if they were simple layers.</p>
<p>here, I leave the complete code:</p>
<pre><code class="lang-auto">import torch
import torch.nn as nn
import copy
from torchvision import models
from collections import OrderedDict

# Custom class for performing double convolutions with group_size
class ConvolutionalBlock(nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0, groups=2, filter_values=None, bias=True):
        super(ConvolutionalBlock, self).__init__()
        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding, groups=groups, bias=bias)

        if filter_values is not None:
            self.initialize_manual_weights(filter_values)
        else:
            self.initialize_integer_weights()
            
        # Set the weight and bias to float16 after initialization
        self.conv.weight.data = self.conv.weight.data.to(torch.float16)
        if self.conv.bias is not None:
            self.conv.bias.data = self.conv.bias.data.to(torch.float16)

    def initialize_integer_weights(self):
        int_weights = torch.randint(-5, 5, self.conv.weight.shape, dtype=torch.int16, device=self.conv.weight.device)
        self.conv.weight.data = int_weights.to(dtype=torch.float16)  # Ensure weights are float16
        if self.conv.bias is not None:
            int_bias = torch.zeros(self.conv.bias.shape, dtype=torch.int16, device=self.conv.bias.device)
            self.conv.bias.data = int_bias.to(dtype=torch.float16)  # Ensure bias is also float16

    def initialize_manual_weights(self, filter_values):
        filter_tensor = torch.tensor(filter_values, dtype=torch.float16, device=self.conv.weight.device).view(self.conv.weight.shape)
        self.conv.weight.data = filter_tensor  # Weights are already float16
        if self.conv.bias is not None:
            self.conv.bias.data.fill_(0.0)
            self.conv.bias.data = self.conv.bias.data.to(dtype=torch.float16)  # Ensure bias is float16

    def forward(self, x):
        return self.conv(x)

# Function to copy and modify the model
def copy_and_modify_cnn_model(model):
    modified_layers = OrderedDict()
    
    for name, layer in model.named_children():
        if (isinstance(layer, nn.Sequential) or list(layer.children())):
            for subname, sublayer in layer.named_children():

                if isinstance(sublayer, nn.Conv2d):
                    in_channels = sublayer.in_channels
                    out_channels = sublayer.out_channels
                    kernel_size = sublayer.kernel_size
                    stride = sublayer.stride
                    padding = sublayer.padding
                    
                    modified_layers[subname] = ConvolutionalBlock(
                        in_channels=in_channels * 2,
                        out_channels=out_channels,
                        kernel_size=kernel_size,
                        stride=stride,
                        padding=padding,
                        groups=2
                    )
                else:
                        modified_layers[subname + "_1"] = copy.deepcopy(sublayer)
                        modified_layers[subname + "_2"] = copy.deepcopy(sublayer)

        else:
            if isinstance(layer, nn.Conv2d):
                in_channels = layer.in_channels
                out_channels = layer.out_channels
                kernel_size = layer.kernel_size
                stride = layer.stride
                padding = layer.padding
                
                modified_layers[name] = ConvolutionalBlock(
                    in_channels=in_channels * 2,
                    out_channels=out_channels,
                    kernel_size=kernel_size,
                    stride=stride,
                    padding=padding,
                    groups=2
                )
            else:
                    modified_layers[name + "_1"] = copy.deepcopy(layer)
                    modified_layers[name + "_2"] = copy.deepcopy(layer)
    
    new_model = nn.Sequential(modified_layers)
    return new_model

# Load a pre-trained model
original_model = models.vgg16(weights=models.VGG16_Weights.IMAGENET1K_V1)

# Set the random seed for reproducibility
torch.manual_seed(0)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False

# Move the model to the GPU if available
device = 'cuda' if torch.cuda.is_available() else 'cpu'
original_model.to(device)

# Create the modified model
new_model = copy_and_modify_cnn_model(original_model).to(device).half()  # Convert the model to float16


# Check the new model
print("modelo original")
print(original_model)
print("nuevo modelo ")
print(new_model)

# Create a synthetic input tensor
input_tensor_1 = torch.randn(1, 3, 224, 224, device=device)  # Original input tensor with 3 channels
synthetic_input_tensor = torch.cat((input_tensor_1, input_tensor_1), dim=1).to(torch.float16)

# Test the output of the modified model
output_modified = new_model(synthetic_input_tensor)
print("\nOutput of the modified model:")
print(output_modified)

</code></pre>
<p>Finally, I leave examples when trying to execute code for a model such as resnet18 or vgg16 and where you can see that when trying to duplicate the model there are layers that it does not detect or groupings of layers that it is not able to detect as loose layers.</p>
<p>VGG16 original layers:</p>
<blockquote>
<p>VGG(<br>
(features): Sequential(<br>
(0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))<br>
(1): ReLU(inplace=True)<br>
(2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))<br>
(3): ReLU(inplace=True)<br>
(4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)<br>
(5): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))<br>
(6): ReLU(inplace=True)<br>
(7): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))<br>
(8): ReLU(inplace=True)<br>
(9): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)<br>
(10): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))<br>
(11): ReLU(inplace=True)<br>
(12): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))<br>
(13): ReLU(inplace=True)<br>
(14): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))<br>
(15): ReLU(inplace=True)<br>
(16): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)<br>
(17): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))<br>
(18): ReLU(inplace=True)<br>
(19): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))<br>
(20): ReLU(inplace=True)<br>
(21): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))<br>
(22): ReLU(inplace=True)<br>
(23): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)<br>
(24): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))<br>
(25): ReLU(inplace=True)<br>
(26): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))<br>
(27): ReLU(inplace=True)<br>
(28): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))<br>
(29): ReLU(inplace=True)<br>
(30): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)<br>
)<br>
(avgpool): AdaptiveAvgPool2d(output_size=(7, 7))<br>
(classifier): Sequential(<br>
(0): Linear(in_features=25088, out_features=4096, bias=True)<br>
(1): ReLU(inplace=True)<br>
(2): Dropout(p=0.5, inplace=False)<br>
(3): Linear(in_features=4096, out_features=4096, bias=True)<br>
(4): ReLU(inplace=True)<br>
(5): Dropout(p=0.5, inplace=False)<br>
(6): Linear(in_features=4096, out_features=1000, bias=True)<br>
)<br>
)</p>
</blockquote>
<p>VGG16 duplicate:</p>
<blockquote>
<p>Sequential(<br>
(0): ConvolutionalBlock(<br>
(conv): Conv2d(6, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=2)<br>
)<br>
(1_1): ReLU(inplace=True)<br>
(1_2): ReLU(inplace=True)<br>
(2): ConvolutionalBlock(<br>
(conv): Conv2d(128, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=2)<br>
)<br>
(3_1): Linear(in_features=4096, out_features=4096, bias=True)<br>
(3_2): Linear(in_features=4096, out_features=4096, bias=True)<br>
(4_1): ReLU(inplace=True)<br>
(4_2): ReLU(inplace=True)<br>
(5): ConvolutionalBlock(<br>
(conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=2)<br>
)<br>
(6_1): Linear(in_features=4096, out_features=1000, bias=True)<br>
(6_2): Linear(in_features=4096, out_features=1000, bias=True)<br>
(7): ConvolutionalBlock(<br>
(conv): Conv2d(256, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=2)<br>
)<br>
(8_1): ReLU(inplace=True)<br>
(8_2): ReLU(inplace=True)<br>
(9_1): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)<br>
(9_2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)<br>
(10): ConvolutionalBlock(<br>
(conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=2)<br>
)<br>
(11_1): ReLU(inplace=True)<br>
(11_2): ReLU(inplace=True)<br>
(12): ConvolutionalBlock(<br>
(conv): Conv2d(512, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=2)<br>
)<br>
(13_1): ReLU(inplace=True)<br>
(13_2): ReLU(inplace=True)<br>
(14): ConvolutionalBlock(<br>
(conv): Conv2d(512, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=2)<br>
)<br>
(15_1): ReLU(inplace=True)<br>
(15_2): ReLU(inplace=True)<br>
(16_1): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)<br>
(16_2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)<br>
(17): ConvolutionalBlock(<br>
(conv): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=2)<br>
)<br>
(18_1): ReLU(inplace=True)<br>
(18_2): ReLU(inplace=True)<br>
(19): ConvolutionalBlock(<br>
(conv): Conv2d(1024, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=2)<br>
)<br>
(20_1): ReLU(inplace=True)<br>
(20_2): ReLU(inplace=True)<br>
(21): ConvolutionalBlock(<br>
(conv): Conv2d(1024, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=2)<br>
)<br>
(22_1): ReLU(inplace=True)<br>
(22_2): ReLU(inplace=True)<br>
(23_1): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)<br>
(23_2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)<br>
(24): ConvolutionalBlock(<br>
(conv): Conv2d(1024, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=2)<br>
)<br>
(25_1): ReLU(inplace=True)<br>
(25_2): ReLU(inplace=True)<br>
(26): ConvolutionalBlock(<br>
(conv): Conv2d(1024, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=2)<br>
)<br>
(27_1): ReLU(inplace=True)<br>
(27_2): ReLU(inplace=True)<br>
(28): ConvolutionalBlock(<br>
(conv): Conv2d(1024, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=2)<br>
)<br>
(29_1): ReLU(inplace=True)<br>
(29_2): ReLU(inplace=True)<br>
(30_1): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)<br>
(30_2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)<br>
(avgpool_1): AdaptiveAvgPool2d(output_size=(7, 7))<br>
(avgpool_2): AdaptiveAvgPool2d(output_size=(7, 7))<br>
(0_1): Linear(in_features=25088, out_features=4096, bias=True)<br>
(0_2): Linear(in_features=25088, out_features=4096, bias=True)<br>
(2_1): Dropout(p=0.5, inplace=False)<br>
(2_2): Dropout(p=0.5, inplace=False)<br>
(5_1): Dropout(p=0.5, inplace=False)<br>
(5_2): Dropout(p=0.5, inplace=False)<br>
)</p>
</blockquote>
<p>In this case, layers 1,3,4,6 of the output are not detected</p>
<p>ResNet18 original layers :</p>
<blockquote>
<p>ResNet(<br>
(conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)<br>
(bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
(relu): ReLU(inplace=True)<br>
(maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)<br>
(layer1): Sequential(<br>
(0): BasicBlock(<br>
(conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)<br>
(bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
(relu): ReLU(inplace=True)<br>
(conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)<br>
(bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
)<br>
(1): BasicBlock(<br>
(conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)<br>
(bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
(relu): ReLU(inplace=True)<br>
(conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)<br>
(bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
)<br>
)<br>
(layer2): Sequential(<br>
(0): BasicBlock(<br>
(conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)<br>
(bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
(relu): ReLU(inplace=True)<br>
(conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)<br>
(bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
(downsample): Sequential(<br>
(0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)<br>
(1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
)<br>
)<br>
(1): BasicBlock(<br>
(conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)<br>
(bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
(relu): ReLU(inplace=True)<br>
(conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)<br>
(bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, aoutput<br>
(layer4): Sequential(<br>
(0): BasicBlock(<br>
(conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)<br>
(bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
(relu): ReLU(inplace=True)<br>
(conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)<br>
(bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
(downsample): Sequential(<br>
(0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)<br>
(1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
)<br>
)<br>
(1): BasicBlock(<br>
(conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)<br>
(bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
(relu): ReLU(inplace=True)<br>
(conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)<br>
(bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
)<br>
)<br>
(avgpool): AdaptiveAvgPool2d(output_size=(1, 1))<br>
(fc): Linear(in_features=512, out_features=1000, bias=True)<br>
)</p>
</blockquote>
<p>ResNet18 duplicate:</p>
<blockquote>
<p>Sequential(<br>
(conv1): ConvolutionalBlock(<br>
(conv): Conv2d(6, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=2)<br>
)<br>
(bn1_1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
(bn1_2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
(relu_1): ReLU(inplace=True)<br>
(relu_2): ReLU(inplace=True)<br>
(maxpool_1): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)<br>
(maxpool_2): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)<br>
(0_1): BasicBlock(<br>
(conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)<br>
(bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
(relu): ReLU(inplace=True)<br>
(conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)<br>
(bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
(downsample): Sequential(<br>
(0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)<br>
(1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
)<br>
)<br>
(0_2): BasicBlock(<br>
(conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)<br>
(bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
(relu): ReLU(inplace=True)<br>
(conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)<br>
(bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
(downsample): Sequential(<br>
(0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)<br>
(1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
)<br>
)<br>
(1_1): BasicBlock(<br>
(conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)<br>
(bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
(relu): ReLU(inplace=True)<br>
(conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)<br>
(bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
)<br>
(1_2): BasicBlock(<br>
(conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)<br>
(bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
(relu): ReLU(inplace=True)<br>
(conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)<br>
(bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)<br>
)<br>
(avgpool_1): AdaptiveAvgPool2d(output_size=(1, 1))<br>
(avgpool_2): AdaptiveAvgPool2d(output_size=(1, 1))<br>
(fc_1): Linear(in_features=512, out_features=1000, bias=True)<br>
(fc_2): Linear(in_features=512, out_features=1000, bias=True)</p>
</blockquote>
<p>In this case, which is more complicated, layers are also lost, and others are copied together as blocks and not as individual layers.</p>
<p>My main question is, how can I copy the topology of any pre-trained model layer by layer, regardless of how it is defined?</p>