Multitaskmodel with pretrained weights has extremely low accuracy

Murphy: <p>Greetings,</p>
<p>Im currently working with this paper: <a href="https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0248414" class="inline-onebox" rel="noopener nofollow ugc">Compare the performance of the models in art classification</a></p>
<p>It uses the following dataset: <a href="http://www.ivl.disco.unimib.it/activities/paintings/" class="inline-onebox" rel="noopener nofollow ugc">Multitask Painting Categorization by Deep Multibranch Neural Network | Imaging and Vision Laboratory</a></p>
<p>I got 3 task, with 100k samples and those have 1508 artists, 125 styles and 41 genres as classes.</p>
<p>My Program is as followed for Resnet multitask implementation:</p>
<pre><code class="lang-auto">

mean = np.array([0.485, 0.456, 0.406])
std = np.array( [0.229, 0.224, 0.225])


# Device configuration
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

data_transforms = {
    'train': transforms.Compose([
        #transforms.ToPILImage(),
        transforms.Resize((256,256)),
        transforms.RandomResizedCrop(224),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize(mean, std)
    ]),
    'test': transforms.Compose([
        transforms.ToPILImage(),
        transforms.Resize(256),
        transforms.CenterCrop(224),
        transforms.ToTensor(),
        transforms.Normalize(mean, std)
    ]),
      'val': 
      transforms.Compose([
        transforms.ToPILImage(),
        transforms.Resize((224, 224)),
        #transforms.RandomResizedCrop(224),
        transforms.ToTensor(),
        transforms.Normalize(mean, std)
    ]),
}

class Resnet18_multiTaskNet(nn.Module):
    def __init__(self, pretrained=True, frozen_feature_layers = False):
        super().__init__()
        
        self.resnet18 = models.resnet18(pretrained=pretrained)
        self.features = nn.Sequential(*list(self.resnet18.children())[:-1]).cuda()
        self.is_frozen = frozen_feature_layers

        if frozen_feature_layers:
            self.freeze_feature_layers()

        # now lets add our new layers 
        in_features = self.resnet18.fc.in_features
        #self.fc0 = nn.Linear(in_features, 512)
        #self.bn_pu = nn.BatchNorm1d(in_features, eps = 1e-5)
        # our five new heads for 5 tasks we have at hand!
        self.fc_artist = nn.Linear(in_features, class_length ['artist']) 
        self.fc_style = nn.Linear(in_features, class_length ['style'])
        self.fc_genre = nn.Linear(in_features, class_length ['genre'])
        torch.nn.init.normal_(self.fc_artist .weight, mean=0.0, std=0.01)
        torch.nn.init.normal_(self.fc_style.weight, mean=0.0, std=0.01)
        torch.nn.init.normal_(self.fc_genre.weight, mean=0.0, std=0.01)


    def forward(self, input_imgs):
        output = self.features(input_imgs)
        #output = output.view(input_imgs.size(0), -1)
        #output = self.bn_pu(F.relu(self.fc0(output)))
        # since color is multi label we should use sigmoid
        # but since we want a numerical stable one, we use
        # nn.BCEWithLogitsloss, as a loss which itself applies sigmoid
        # and thus accepts logits. so we wont use sigmoid here for that matter
        # its much stabler than sigmoid+BCE
        print(output.shape)
        output = torch.flatten(output, 1)
        print(output.shape)
        prd_arist = self.fc_artist(output)
        prd_style = self.fc_style(output)
        prd_genre = self.fc_genre(output)

        
        return prd_arist, prd_style, prd_genre
    
    def _set_freeze_(self, status):
        for n,p in self.features.named_parameters():
            p.requires_grad = status
        # for m in self.features.children():
        #     for p in m.parameters():
        #         p.requires_grad=status    


    def freeze_feature_layers(self):
        self._set_freeze_(False)

    def unfreeze_feature_layers(self):
        self._set_freeze_(True)
</code></pre>
<pre><code class="lang-auto">
class customdataset(Dataset):
    def __init__(self, csv_file, root_dir, transform, n =None):
        if n == None:
            df = pd.read_csv(csv_file)
        else:
            df = pd.read_csv(csv_file, nrows=n)
        unsorted_labels = {x: df[x].unique() for x in ['artist','style','genre']}
        self.labels = {x: np.sort(unsorted_labels[x]) for x in ['artist','style','genre']}
        self.annotations = df
        self.root_dir = root_dir
        self.transform = transform

    def __len__(self):
        return len(self.annotations)

    def __getitem__(self, index):
        img_path = os.path.join(self.root_dir, self.annotations.iloc[index, 0])
        image = Image.open(img_path).convert('RGB')

        image_artist = self.annotations.iloc[index,1]
        image_style = self.annotations.iloc[index,2]
        image_genre = self.annotations.iloc[index,3]

        #test = le.transform(list(le.classes_))
        #le fit anpassen an die unique df col
        le = preprocessing.LabelEncoder()
        le.fit(self.labels['style'])
        image_style = le.transform([image_style])
        le.fit(self.labels['artist'])
        image_artist = le.transform([image_artist])
        le.fit(self.labels['genre'])
        image_genre = le.transform([image_genre])
    
        if self.transform:
            #image =  np.array(image)
            #size = image.shape
            #print(size)
            #if(size[2] == 1):
                #image = gray2rgb(image)
            image = self.transform(image)
        
        return(image,image_artist,image_style,image_genre)
</code></pre>
<pre><code class="lang-auto">def train_model(model, criterion, optimizer, scheduler, num_epochs):
    since = time.time()

    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0

    fields = ['artist', 'style', 'genre']
    #accuracies = [0.0]*len(fields)

    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('\nLearning rate at this epoch is: %0.9f' % scheduler.get_lr()[0])
        print('\nLearning rate at this epoch is: %.9f ', optimizer.param_groups[0]['lr'], '\n') 
        print('-' * 10)
        lrs = []

        # Each epoch has a training and validation phase
        for phase in ['train', 'val']:
            if phase == 'train':
                model.train()  # Set model to training mode
            else:
                model.eval()   # Set model to evaluate mode

            running_loss = 0.0
            running_loss2 = 0.0
            running_loss3 = 0.0
            running_loss4 = 0.0
            running_loss5 = 0.0

            running_corrects_artist = 0
            running_corrects_style = 0
            running_corrects_genre = 0

            # Iterate over data.
            i = 0
            for inputs,image_artist,image_style,image_genre in dataloaders[phase]:
                i = i + 1
                image_style
                inputs = inputs.to(device)
                image_style = image_style.to(device)
                image_artist= image_artist.to(device)
                image_genre = image_genre.to(device)



                # forward
                # track history if only in train
                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model(inputs)

                    #style
                    _, preds_artist = torch.max(outputs[0], 1)
                    image_artist=  torch.squeeze(image_artist).long()
                    loss_artist = criterion[0](outputs[0], image_artist)
                 
                    f1_metric['artist'](preds_artist,image_artist.data)

                    #style
                    _, preds_style = torch.max(outputs[1], 1)
                    image_style=  torch.squeeze(image_style).long()
                    loss_style = criterion[1](outputs[1], image_style)

                    #genre
                    _, preds_genre = torch.max(outputs[2], 1)
                    image_genre=  torch.squeeze(image_genre).long()
                    loss_genre = criterion[2](outputs[2], image_genre)

                 
                    
                    loss_final = loss_artist + loss_style + loss_genre 

                    # backward + optimize only if in training phase
                    if phase == 'train':
                        # zero the parameter gradients
                        optimizer.zero_grad()
                        loss_final.backward()
                        optimizer.step()
    
                # statistics
                running_loss += loss_final.item() * inputs.size(0)
                running_loss2 += loss_final.item() * inputs.size(0)
                running_loss3 += loss_artist * inputs.size(0)
                running_loss4 += loss_style * inputs.size(0)
                running_loss5 += loss_genre * inputs.size(0)


                #Accuarcy corrects
                running_corrects_artist += torch.mean((preds_artist.view(*image_artist.shape) == image_artist).float())
                running_corrects_style += torch.mean((preds_style.view(*image_style.shape) == image_style).float())
                running_corrects_genre += torch.mean((preds_genre.view(*image_genre.shape) == image_genre).float())

                running_corrects_artis_2 = torch.mean((preds_artist.view(*image_artist.shape) == image_artist).float())
                print(f'\n running_corrects_artis_2: {running_corrects_artis_2}',)

                running_corrects_style_2 = torch.mean((preds_style.view(*image_style.shape) == image_style).float())
                print(f' running_corrects_style_2: {running_corrects_style_2}',)

                running_corrects_genre_2 = torch.mean((preds_genre.view(*image_genre.shape) == image_genre).float())
                print(f' running_corrects_genre_2: {running_corrects_genre_2}  \n',)

                #F1-Score
                current_f1_artist = f1_metric['artist'](preds_artist,image_artist.data)
                current_f1_style = f1_metric['style'](preds_style,image_style.data)
                current_f1_genre = f1_metric['genre'](preds_genre,image_genre.data)

                #Plot
                lrs.append(torch.mean((preds_artist.view(*image_artist.shape) == image_artist).float()).cpu())
                if(i) % 10 == 0:
                    plt.plot(lrs)
                    plt.savefig('foo.png')


                if (i+1) % 1 == 0:
                    print (f'Epoch [{epoch+1}/{num_epochs}], Step [{i+1}/{n_total_steps}], Loss: {loss_final.item():.4f} ,Running Loss: {running_loss / dataset_sizes[phase]}, Running Loss for Batch: {running_loss2/ 80} ')
                    #print(f'stylepred: {preds_style} , styleimage: {image_style.data}')
                    print(f'running_corrects_artist: {running_corrects_artist} , running_corrects_style: {running_corrects_style}, running_corrects_genre: {running_corrects_genre}')
                    print(f'loss_final.item(): {loss_final.item()} ')
                    print(f'loss_artist: {loss_artist.item()} , loss_style: {loss_style.item()}, loss_gemre: {loss_genre.item()} ')
                    print(f'loss_artist  * inputs.size(0): {loss_artist.item()  * inputs.size(0)} , loss_style: {loss_style.item()  * inputs.size(0)}, loss_gemre: {loss_genre.item()  * inputs.size(0)} ')

                    running_loss2 = 0.0

            if phase == 'train':
                scheduler.step()

            #Losses
            epoch_loss = running_loss / dataset_sizes[phase]
            epoch_loss_artist = running_loss3  / dataset_sizes[phase]
            epoch_loss_style = running_loss4  / dataset_sizes[phase]
            epoch_loss_genre = running_loss5  / dataset_sizes[phase]

            #Accuracys
            epoch_acc_artist = running_corrects_artist.double() / dataset_sizes[phase]
            epoch_acc_style = running_corrects_style.double() / dataset_sizes[phase]
            epoch_acc_genre = running_corrects_genre.double() / dataset_sizes[phase]

        

            print()
            print('{} artist_Acc: {:.4f} '
                  'style_acc: {:.4f}  '
                  'genre_acc: {:.4f}  '.format(
                phase, epoch_acc_artist ,epoch_acc_style,epoch_acc_genre))
            print('{} total loss: {:.4f} artist loss: {:.4f} style loss: {:.4f} genre loss: {:.4f}'.format(phase,epoch_loss,epoch_loss_artist ,
                                                                                                          epoch_loss_style,epoch_loss_genre,
                                                                                                          ))
            print()

            # deep copy the model
            if phase == 'val' and epoch_acc_genre  &gt; best_acc:
                best_acc = epoch_acc_genre 
                best_model_wts = copy.deepcopy(model.state_dict())

        print()

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(
        time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))
    print('F1-Score:')
    print(f'Artist: {current_f1_artist}')
    print(f'Style: {current_f1_style}')
    print(f'Genre: {current_f1_genre}')

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model
</code></pre>
<p>I currently train everything on 3 Machines, the VGG-Model gets only 30% and transformer too. So i must make  big mistakes. I have no idea where, because in the journal they hit about 60-80% accuracy with the same Data. The models start in each category with around 0% and they need several epochs go get better for example they need 20 epochs to make it from 0 to 8% or something.</p>
<p>They used a 0.2 learning rate from what i could gather from the studies and they implement also the coisinelearningscheduler. with SGD. They used 160 epochs and i need about 1 week to make 160 epochs currently.</p>
<p>I think my class for the model is maybe the problem</p>

eqy: <pre><code class="lang-auto">        torch.nn.init.normal_(self.fc_artist .weight, mean=0.0, std=0.01)
        torch.nn.init.normal_(self.fc_style.weight, mean=0.0, std=0.01)
        torch.nn.init.normal_(self.fc_genre.weight, mean=0.0, std=0.01)
</code></pre>
<p>This looks very suspicious, why do you need a custom initialization with very small std? I would check if the default initialization at least brings faster convergence.</p>
<p>I would also investigate if different learning rate decay schedules achieve different results.</p>

Murphy: <p>greetings i tested alot of things:</p>
<p>i trained the models speretaly and it seems they work without the multitask.</p>
<p>I also took your advice and did not make a custom initialization. It works good but the problem with multitask is still there:</p>
<p>I think its because of the loss function.</p>
<p>Currently i just add the loss together, put the sum in backprop. But that doesnt seem to work. Is there a good tutorial or way to make the loss for multitask pretrained models?</p>
<p>I integrated the Multilossfunction from this thread:</p><aside class="quote quote-modified" data-post="18" data-topic="39681">
  <div class="title">
    <div class="quote-controls"></div>
    <img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/tony-y/48/8373_2.png" class="avatar">
    <a href="https://discuss.pytorch.org/t/how-to-learn-the-weights-between-two-losses/39681/18">How to learn the weights between two losses?</a> 
  </div>
  <blockquote>
    This paper proposes total loss composed of MSE and CrossEntropy losses. Other losses are outside the scope of the assumption. An implementation for Equation (10) where y1 is a continuous output and y2 is a discrete output: 
import torch
import torch.nn as nn
import torch.optim as optim

class MultiTaskLoss(nn.Module):
    def __init__(self, model, loss_fn, eta):
        super(MultiTaskLoss, self).__init__()
        self.model = model
        self.loss_fn = loss_fn
        self.eta = nn.Parameter…
  </blockquote>
</aside>

<p>But it still does not work. I can train them seperately but together they dont work (one task accurcay rises, while the other stays low)</p>
<p>I use crossentropy for both.</p>
<pre><code class="lang-auto">model = Resnet50_multiTaskNet().to(device)        
criterion = [nn.CrossEntropyLoss(), nn.CrossEntropyLoss()]

def loss_fn1(x, cls):
    return 2 * criterion[0](x, cls)
def loss_fn2(x, cls):
    return 2 * criterion[1](x, cls)

mtl = MultiTaskLoss(model=model,
                    loss_fn=[loss_fn1, loss_fn2],
                    eta=[1.0, 1.0]).to(device)  

optimizer = optim.Adam(mtl.parameters())

class Resnet50_multiTaskNet(nn.Module):
    def __init__(self):
        super(Resnet50_multiTaskNet, self).__init__()
        
        self.model =  models.resnet50(weights=ResNet50_Weights.IMAGENET1K_V2)

        for param in self.model.parameters():
            param.requires_grad = False 

        self.fc_artist = nn.Linear(2048, class_length ['artist']).to(device)
        self.fc_style = nn.Linear(2048, class_length ['style']).to(device)

    def forward(self, x):
        x = self.model.conv1(x)
        x = self.model.bn1(x)
        x = self.model.relu(x)
        x = self.model.maxpool(x)

        x = self.model.layer1(x)
        x = self.model.layer2(x)
        x = self.model.layer3(x)
        x = self.model.layer4(x)
        x = self.model.avgpool(x)
        x = x.view(x.size(0), -1)

        x_artist = self.fc_artist(x)
        x_style = self.fc_style(x)
        return x_artist, x_style
    
#multitaskloss
class MultiTaskLoss(nn.Module):
    def __init__(self, model, loss_fn, eta) -&gt; None:
        super(MultiTaskLoss, self).__init__()
        self.model = model
        self.loss_fn = loss_fn
        self.eta = nn.Parameter(torch.Tensor(eta))

    def forward(self, input, targets) -&gt; Tuple[torch.Tensor, torch.Tensor]:
        outputs = self.model(input)
        loss = [l(o,y) for l, o, y in zip(self.loss_fn, outputs, targets)]
        total_loss = torch.stack(loss) * torch.exp(-self.eta) + self.eta
        return loss, total_loss.sum(), outputs  # omit 1/2
</code></pre>
<p>Anyone has an idea why?</p>
<p>I had the idea to use backprop 2 times, while freezing every except the fc. So i train the fc for task1 while task 2 and the rest is frozen and then task 2 while task1 and the rest is frozen.</p>