code/recognization/model.py

import torch

from .feature_extraction import HRNet_FeatureExtractor
from .sequence_modeling import BidirectionalLSTM
from .dropout_layer import dropout_layer
from .prediction import Attention
import torch.nn as nn

# Other CNN Architectures
from .feature_extraction import DenseNet_FeatureExtractor, InceptionUNet_FeatureExtractor
from .feature_extraction import RCNN_FeatureExtractor, ResNet_FeatureExtractor
from .feature_extraction import ResUnet_FeatureExtractor, AttnUNet_FeatureExtractor
from .feature_extraction import UNet_FeatureExtractor, UNetPlusPlus_FeatureExtractor
from .feature_extraction import VGG_FeatureExtractor

# Other sequential models
from .sequence_modeling import LSTM, GRU, MDLSTM

class Text_recognization_model(nn.Module):

    """ The constractor init the struture of the model """
    def __init__(self, opt):
        super(Text_recognization_model, self).__init__()
        # opt is the configration of the model
        self.opt = opt
        # The model consist of three stages
        # FeatureExtraction, SequenceModeling and Prediction
        self.stages = {'Feat': opt.FeatureExtraction,
                       'Seq': opt.SequenceModeling,
                       'Pred': opt.Prediction}

        """ FeatureExtraction """
        # High-Resolution Network, it maintains high-resolution feature maps
        if opt.FeatureExtraction == 'HRNet':
            self.FeatureExtraction = HRNet_FeatureExtractor(opt.input_channel, opt.output_channel )
        elif opt.FeatureExtraction == 'Densenet':
            self.FeatureExtraction = DenseNet_FeatureExtractor(opt.input_channel, opt.output_channel)
        elif opt.FeatureExtraction == 'InceptionUnet':
            self.FeatureExtraction = InceptionUNet_FeatureExtractor(opt.input_channel, opt.output_channel)
        elif opt.FeatureExtraction == 'RCNN':
            self.FeatureExtraction = RCNN_FeatureExtractor(opt.input_channel, opt.output_channel)
        elif opt.FeatureExtraction == 'ResNet':
            self.FeatureExtraction = ResNet_FeatureExtractor(opt.input_channel, opt.output_channel)
        elif opt.FeatureExtraction == 'ResUnet':
            self.FeatureExtraction = ResUnet_FeatureExtractor(opt.input_channel, opt.output_channel)
        elif opt.FeatureExtraction == 'AttnUNet':
            self.FeatureExtraction = AttnUNet_FeatureExtractor(opt.input_channel, opt.output_channel)
        elif opt.FeatureExtraction == 'UNet':
            self.FeatureExtraction = UNet_FeatureExtractor(opt.input_channel, opt.output_channel)
        elif opt.FeatureExtraction == 'UnetPlusPlus':
            self.FeatureExtraction = UNetPlusPlus_FeatureExtractor(opt.input_channel, opt.output_channel)
        elif opt.FeatureExtraction == 'VGG':
            self.FeatureExtraction = VGG_FeatureExtractor(opt.input_channel, opt.output_channel)
        else:
            raise Exception('No FeatureExtraction module specified')
        self.FeatureExtraction_output = opt.output_channel
        self.AdaptiveAvgPool = nn.AdaptiveAvgPool2d((None, 1)) # Transform final (imgH/16-1) -> 1
        
        """
        Temporal Dropout
        """
        self.dropout1 = dropout_layer(opt.device)
        self.dropout2 = dropout_layer(opt.device)
        self.dropout3 = dropout_layer(opt.device)
        self.dropout4 = dropout_layer(opt.device)
        self.dropout5 = dropout_layer(opt.device)

        """ Sequence modeling"""
        if opt.SequenceModeling == 'LSTM':
            self.SequenceModeling = LSTM(self.FeatureExtraction_output, opt.hidden_size, opt.hidden_size)
        elif opt.SequenceModeling == 'GRU':
            self.SequenceModeling = GRU(self.FeatureExtraction_output, opt.hidden_size, opt.hidden_size)
        elif opt.SequenceModeling == 'MDLSTM':
            self.SequenceModeling = MDLSTM(self.FeatureExtraction_output, opt.hidden_size, opt.hidden_size)
        elif opt.SequenceModeling == 'BiLSTM':
            self.SequenceModeling = BidirectionalLSTM(self.FeatureExtraction_output, opt.hidden_size, opt.hidden_size)
        # Double BiLSTM 
        elif opt.SequenceModeling == 'DBiLSTM':
            self.SequenceModeling = nn.Sequential(
                BidirectionalLSTM(self.FeatureExtraction_output, opt.hidden_size, opt.hidden_size),
                BidirectionalLSTM(opt.hidden_size, opt.hidden_size, opt.hidden_size))
        else:
            raise Exception('No Sequence Modeling module specified')
        self.SequenceModeling_output = opt.hidden_size

        """ Prediction """
        if opt.Prediction == 'CTC':
            self.Prediction = nn.Linear(self.SequenceModeling_output, opt.num_class)
        elif opt.Prediction == 'Attn':
            self.Prediction = Attention(self.SequenceModeling_output, opt.hidden_size, opt.num_class, opt.device)
        else:
            raise Exception('Prediction is neither CTC or Attn')

    def forward(self, input, text=None, is_train=True):
        """ Feature extraction stage """
        ### Pass input to the feature extraction network ###
        visual_feature = self.FeatureExtraction(input)
        # print(visual_feature.shape) # [32, 32, 32, 400] #HRNet, [32, 512, 32, 400] #UNet
        ### Then make pooling ###
        visual_feature = self.AdaptiveAvgPool(visual_feature.permute(0, 3, 1, 2))  # [b, c, h, w] -> [b, w, c, h]
        # print(visual_feature.shape) # [32, 400, 32, 1] #HRNet, [32, 400, 512, 1] #UNet
        ### Remove the columb 3 Ex=> [32,400,32,1] will be [32,400,32] ###
        visual_feature = visual_feature.squeeze(3)
        # print(visual_feature.shape) # [32, 400, 32] #HRNet, [32, 400, 512] #UNet

        
        """ Temporal Dropout + Sequence modeling stage """
        # contextual_feature = self.SequenceModeling(visual_feature) ##### Without temporal dropout
        if (self.training):
            visual_feature_after_dropout1 = self.dropout1(visual_feature)
            contextual_feature = self.SequenceModeling(visual_feature_after_dropout1)
        else :
            # Inference Phase, make multiple dropout, and take the average of them, this  is called Monte Carlo Dropout 
            visual_feature_after_dropout1 = self.dropout1(visual_feature)
            visual_feature_after_dropout2 = self.dropout2(visual_feature)
            visual_feature_after_dropout3 = self.dropout3(visual_feature)
            visual_feature_after_dropout4 = self.dropout4(visual_feature)
            visual_feature_after_dropout5 = self.dropout5(visual_feature)
            contextual_feature1 = self.SequenceModeling(visual_feature_after_dropout1)
            contextual_feature2 = self.SequenceModeling(visual_feature_after_dropout2)
            contextual_feature3 = self.SequenceModeling(visual_feature_after_dropout3)
            contextual_feature4 = self.SequenceModeling(visual_feature_after_dropout4)
            contextual_feature5 = self.SequenceModeling(visual_feature_after_dropout5)
            contextual_feature =  ( (contextual_feature1).add ((contextual_feature2).add(((contextual_feature3).add(((contextual_feature4).add(contextual_feature5)))))) ) * (1/5)

        """ Prediction stage """
        if self.stages['Pred'] == 'CTC':
            prediction = self.Prediction(contextual_feature.contiguous())
        else:
            if text is None:
                raise Exception('Input text (for prediction) to model is None')
            text = text.to(self.opt.device)
            prediction = self.Prediction(contextual_feature, text, is_train, batch_max_length=self.opt.batch_max_length)
        
        return prediction