Repository: openspeech-team/openspeech (language: python)

Issue:
Add ContextNet Model
# 🚀 Feature request



## Motivation

[ContextNet: Improving Convolutional Neural Networks for Automatic Speech Recognition with Global Context](https://arxiv.org/pdf/2005.03191.pdf)

## Your contribution

- implementation

- validation



Discussion / hints:
Record the branch you work on.
https://github.com/sooftware/OpenSpeech/tree/0.2.1_ha

Fix patch:
diff --git a/openspeech/encoders/__init__.py b/openspeech/encoders/__init__.py
index ae2be71..4d942d7 100644
--- a/openspeech/encoders/__init__.py
+++ b/openspeech/encoders/__init__.py
@@ -22,6 +22,7 @@
 
 from .openspeech_encoder import OpenspeechEncoder
 from .conformer_encoder import ConformerEncoder
+from .contextnet_encoder import ContextNetEncoder
 from .convolutional_lstm_encoder import ConvolutionalLSTMEncoder
 from .convolutional_transformer_encoder import ConvolutionalTransformerEncoder
 from .lstm_encoder import LSTMEncoder
diff --git a/openspeech/encoders/conformer_encoder.py b/openspeech/encoders/conformer_encoder.py
index 37df7a2..25aabfd 100644
--- a/openspeech/encoders/conformer_encoder.py
+++ b/openspeech/encoders/conformer_encoder.py
@@ -48,7 +48,7 @@ class ConformerEncoder(OpenspeechEncoder):
         conv_dropout_p (float, optional): Probability of conformer convolution module dropout
         conv_kernel_size (int or tuple, optional): Size of the convolving kernel
         half_step_residual (bool): Flag indication whether to use half step residual or not
-        joint_ctc_attention (bool, optional): flag indication joint ctc attention or not
+            joint_ctc_attention (bool, optional): flag indication joint ctc attention or not
 
     Inputs: inputs, input_lengths
         - **inputs** (batch, time, dim): Tensor containing input vector
diff --git a/openspeech/encoders/contextnet_encoder.py b/openspeech/encoders/contextnet_encoder.py
new file mode 100644
index 0000000..76de328
--- /dev/null
+++ b/openspeech/encoders/contextnet_encoder.py
@@ -0,0 +1,123 @@
+# MIT License
+#
+# Copyright (c) 2021 Soohwan Kim and Sangchun Ha and Soyoung Cho
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import torch
+import torch.nn as nn
+from torch import Tensor
+from typing import Tuple
+
+from openspeech.encoders.openspeech_encoder import OpenspeechEncoder
+from openspeech.modules.contextnet_block import ContextNetBlock
+from openspeech.modules import Conv2dSubsampling, Linear, ConformerBlock, Transpose
+
+
+class ContextNetEncoder(OpenspeechEncoder):
+    r"""
+    ContextNetEncoder goes through 23 convolution blocks to convert to higher feature values.
+
+    Args:
+        num_classes (int): Number of classification
+        model_size (str, optional): Size of the model['small', 'medium', 'large'] (default : 'medium')
+        input_dim (int, optional): Dimension of input vector (default : 80)
+        num_layers (int, optional): The number of convolutional layers (default : 5)
+        kernel_size (int, optional): Value of convolution kernel size (default : 5)
+        num_channels (int, optional): The number of channels in the convolution filter (default: 256)
+        output_dim (int, optional): Dimension of encoder output vector (default: 640)
+        joint_ctc_attention (bool, optional): flag indication joint ctc attention or not
+
+    Inputs: inputs, input_lengths
+        - **inputs**: Parsed audio of batch size number `FloatTensor` of size ``(batch, seq_length, dimension)``
+        - **input_lengths**: Tensor representing the sequence length of the input ``(batch)``
+
+    Returns: output, output_lengths
+        - **output**: Tensor of encoder output `FloatTensor` of size
+                ``(batch, seq_length, dimension)``
+        - **encoder_logits**: Log probability of encoders outputs will be passed to CTC Loss.
+                If joint_ctc_attention is False, return None.
+        - **output_lengths**: Tensor representing the length of the encoder output ``(batch)``
+    """
+    supported_models = {
+        'small': 0.5,
+        'medium': 1,
+        'large': 2,
+    }
+
+    def __init__(
+            self,
+            num_classes: int,
+            model_size: str = 'medium',
+            input_dim: int = 80,
+            num_layers: int = 5,
+            kernel_size: int = 5,
+            num_channels: int = 256,
+            output_dim: int = 640,
+            joint_ctc_attention: bool = False,
+    ) -> None:
+        super(ContextNetEncoder, self).__init__()
+        assert model_size in ('small', 'medium', 'large'), f'{model_size} is not supported.'
+
+        alpha = self.supported_models[model_size]
+
+        num_channels = int(num_channels * alpha)
+        output_dim = int(output_dim * alpha)
+
+        self.joint_ctc_attention = joint_ctc_attention
+        self.blocks = ContextNetBlock.make_conv_blocks(input_dim, num_layers, kernel_size, num_channels, output_dim)
+        if self.joint_ctc_attention:
+            self.fc = nn.Linear(output_dim, num_classes, bias=False)
+
+    def forward(
+            self,
+            inputs: Tensor,
+            input_lengths: Tensor,
+    ) -> Tuple[Tensor, Tensor, Tensor]:
+        r"""
+        Forward propagate a `inputs` for audio encoder.
+
+        Args:
+            **inputs** (torch.FloatTensor): Parsed audio of batch size number `FloatTensor` of size
+                ``(batch, seq_length, dimension)``
+            **input_lengths** (torch.LongTensor): Tensor representing the sequence length of the input
+                `LongTensor` of size ``(batch)``
+
+        Returns:
+            **output** (torch.FloatTensor): Tensor of encoder output `FloatTensor` of size
+                ``(batch, seq_length, dimension)``
+            **encoder_logits** (torch.FloatTensor): Log probability of encoders outputs will be passed to CTC Loss.
+                If joint_ctc_attention is False, return None.
+            **output_lengths** (torch.LongTensor): Tensor representing the length of the encoder output
+                `LongTensor` of size ``(batch)``
+        """
+        encoder_logits = None
+
+        output = inputs.transpose(1, 2)
+        output_lengths = input_lengths
+
+        for block in self.blocks:
+            output, output_lengths = block(output, output_lengths)
+
+        output = output.transpose(1, 2)
+
+        if self.joint_ctc_attention:
+            encoder_logits = self.fc(output).log_softmax(dim=2)
+
+        return output, encoder_logits, output_lengths
\ No newline at end of file
diff --git a/openspeech/models/__init__.py b/openspeech/models/__init__.py
index 0ffff15..22aea8c 100644
--- a/openspeech/models/__init__.py
+++ b/openspeech/models/__init__.py
@@ -89,6 +89,12 @@ def register_model_cls(cls):
 from .conformer_lstm.configurations import ConformerLSTMConfigs
 from .conformer_transducer.model import ConformerTransducerModel
 from .conformer_transducer.configurations import ConformerTransducerConfigs
+from .contextnet.model import ContextNetModel
+from .contextnet.configurations import ContextNetConfigs
+from .contextnet_lstm.model import ContextNetLSTMModel
+from .contextnet_lstm.configurations import ContextNetLSTMConfigs
+from .contextnet_transducer.model import ContextNetTransducerModel
+from .contextnet_transducer.configurations import ContextNetTransducerConfigs
 from .deep_cnn_with_joint_ctc_listen_attend_spell.model import DeepCNNWithJointCTCListenAttendSpellModel
 from .deep_cnn_with_joint_ctc_listen_attend_spell.configurations import DeepCNNWithJointCTCListenAttendSpellConfigs
 from .deepspeech2.model import DeepSpeech2Model
@@ -141,6 +147,12 @@ def register_model_cls(cls):
     "ConformerLSTMConfigs",
     "ConformerTransducerModel",
     "ConformerTransducerConfigs",
+    "ContextNetModel",
+    "ContextNetConfigs",
+    "ContextNetLSTMModel",
+    "ContextNetLSTMConfigs",
+    "ContextNetTransducerModel",
+    "ContextNetTransducerConfigs",
     "DeepCNNWithJointCTCListenAttendSpellModel",
     "DeepCNNWithJointCTCListenAttendSpellConfigs",
     "DeepSpeech2Model",
diff --git a/openspeech/models/contextnet/__init__.py b/openspeech/models/contextnet/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/openspeech/models/contextnet/configurations.py b/openspeech/models/contextnet/configurations.py
new file mode 100644
index 0000000..5f99767
--- /dev/null
+++ b/openspeech/models/contextnet/configurations.py
@@ -0,0 +1,71 @@
+# MIT License
+#
+# Copyright (c) 2021 Soohwan Kim and Sangchun Ha and Soyoung Cho
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from dataclasses import dataclass, field
+
+from openspeech.dataclass.configurations import OpenspeechDataclass
+
+
+@dataclass
+class ContextNetConfigs(OpenspeechDataclass):
+    r"""
+    This is the configuration class to store the configuration of
+    a :class:`~openspeech.models.ContextNet`.
+
+    It is used to initiated an `ContextNet` model.
+
+    Configuration objects inherit from :class: `~openspeech.dataclass.configs.OpenspeechDataclass`.
+
+    Args:
+        model_name (str): Model name (default: contextnet)
+        model_size (str, optional): Size of the model['small', 'medium', 'large'] (default : 'medium')
+        input_dim (int, optional): Dimension of input vector (default : 80)
+        num_encoder_layers (int, optional): The number of convolution layers (default : 5)
+        kernel_size (int, optional): Value of convolution kernel size (default : 5)
+        num_channels (int, optional): The number of channels in the convolution filter (default: 256)
+        encoder_dim (int, optional): Dimension of encoder output vector (default: 640)
+        optimizer (str): Optimizer for training. (default: adam)
+    """
+    model_name: str = field(
+        default="contextnet", metadata={"help": "Model name"}
+    )
+    model_size: str = field(
+        default="medium", metadata={"help": "Model size"}
+    )
+    input_dim: int = field(
+        default=80, metadata={"help": "Dimension of input vector"}
+    )
+    num_encoder_layers: int = field(
+        default=5, metadata={"help": "The number of convolution layers"}
+    )
+    kernel_size: int = field(
+        default=5, metadata={"help": "Value of convolution kernel size"}
+    )
+    num_channels: int = field(
+        default=256, metadata={"help": "The number of channels in the convolution filter"}
+    )
+    encoder_dim: int = field(
+        default=640, metadata={"help": "Dimension of encoder output vector"}
+    )
+    optimizer: str = field(
+        default="adam", metadata={"help": "Optimizer for training"}
+    )
diff --git a/openspeech/models/contextnet/model.py b/openspeech/models/contextnet/model.py
new file mode 100644
index 0000000..9029e4e
--- /dev/null
+++ b/openspeech/models/contextnet/model.py
@@ -0,0 +1,147 @@
+# MIT License
+#
+# Copyright (c) 2021 Soohwan Kim and Sangchun Ha and Soyoung Cho
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from omegaconf import DictConfig
+from torch import Tensor
+from typing import Dict
+from collections import OrderedDict
+
+from openspeech.models import register_model
+from openspeech.models import OpenspeechCTCModel
+from openspeech.models.contextnet.configurations import ContextNetConfigs
+from openspeech.encoders import ContextNetEncoder
+from openspeech.modules.wrapper import Linear
+from openspeech.vocabs.vocab import Vocabulary
+
+
+@register_model('contextnet', dataclass=ContextNetConfigs)
+class ContextNetModel(OpenspeechCTCModel):
+    r"""
+    Conformer Encoder Only Model.
+
+    Args:
+        configs (DictConfig): configuration set.
+        vocab (Vocabulary): the class of vocabulary
+
+    Inputs:
+        inputs (torch.FloatTensor): A input sequence passed to encoders. Typically for inputs this will be a padded
+            `FloatTensor` of size ``(batch, seq_length, dimension)``.
+        input_lengths (torch.LongTensor): The length of input tensor. ``(batch)``
+
+    Returns:
+        * dict (dict): Result of model predictions that contains `y_hats`, `logits`, `output_lengths`
+    """
+    def __init__(self, configs: DictConfig, vocab: Vocabulary,) -> None:
+        super(ContextNetModel, self).__init__(configs, vocab)
+        self.fc = Linear(self.configs.model.encoder_dim, self.num_classes, bias=False)
+
+    def build_model(self):
+        self.encoder = ContextNetEncoder(
+            num_classes=self.num_classes,
+            model_size=self.configs.model.model_size,
+            input_dim=self.configs.audio.num_mels,
+            num_layers=self.configs.model.num_encoder_layers,
+            kernel_size=self.configs.model.kernel_size,
+            num_channels=self.configs.model.num_channels,
+            output_dim=self.configs.model.encoder_dim,
+            joint_ctc_attention=False,
+        )
+
+    def forward(self, inputs: Tensor, input_lengths: Tensor) -> Dict[str, Tensor]:
+        r"""
+        Forward propagate a `inputs` and `targets` pair for inference.
+
+        Inputs:
+            inputs (torch.FloatTensor): A input sequence passed to encoders. Typically for inputs this will be a padded
+                `FloatTensor` of size ``(batch, seq_length, dimension)``.
+            input_lengths (torch.LongTensor): The length of input tensor. ``(batch)``
+
+        Returns:
+            * dict (dict): Result of model predictions that contains `y_hats`, `logits`, `output_lengths`
+        """
+        return super(ContextNetModel, self).forward(inputs, input_lengths)
+
+    def training_step(self, batch: tuple, batch_idx: int) -> OrderedDict:
+        r"""
+        Forward propagate a `inputs` and `targets` pair for training.
+
+        Inputs:
+            batch (tuple): A train batch contains `inputs`, `targets`, `input_lengths`, `target_lengths`
+            batch_idx (int): The index of batch
+
+        Returns:
+            loss (torch.Tensor): loss for training
+        """
+        inputs, targets, input_lengths, target_lengths = batch
+        encoder_outputs, encoder_logits, output_lengths = self.encoder(inputs, input_lengths)
+        logits = self.fc(encoder_outputs).log_softmax(dim=-1)
+        return self.collect_outputs(
+            stage='train',
+            logits=logits,
+            output_lengths=output_lengths,
+            targets=targets,
+            target_lengths=target_lengths,
+        )
+
+    def validation_step(self, batch: tuple, batch_idx: int) -> OrderedDict:
+        r"""
+        Forward propagate a `inputs` and `targets` pair for validation.
+
+        Inputs:
+            batch (tuple): A train batch contains `inputs`, `targets`, `input_lengths`, `target_lengths`
+            batch_idx (int): The index of batch
+
+        Returns:
+            loss (torch.Tensor): loss for training
+        """
+        inputs, targets, input_lengths, target_lengths = batch
+        encoder_outputs, encoder_logits, output_lengths = self.encoder(inputs, input_lengths)
+        logits = self.fc(encoder_outputs).log_softmax(dim=-1)
+        return self.collect_outputs(
+            stage='valid',
+            logits=logits,
+            output_lengths=output_lengths,
+            targets=targets,
+            target_lengths=target_lengths,
+        )
+
+    def test_step(self, batch: tuple, batch_idx: int) -> OrderedDict:
+        r"""
+        Forward propagate a `inputs` and `targets` pair for test.
+
+        Inputs:
+            batch (tuple): A train batch contains `inputs`, `targets`, `input_lengths`, `target_lengths`
+            batch_idx (int): The index of batch
+
+        Returns:
+            loss (torch.Tensor): loss for training
+        """
+        inputs, targets, input_lengths, target_lengths = batch
+        encoder_outputs, encoder_logits, output_lengths = self.encoder(inputs, input_lengths)
+        logits = self.fc(encoder_outputs).log_softmax(dim=-1)
+        return self.collect_outputs(
+            stage='test',
+            logits=logits,
+            output_lengths=output_lengths,
+            targets=targets,
+            target_lengths=target_lengths,
+        )
\ No newline at end of file
diff --git a/openspeech/models/contextnet_lstm/__init__.py b/openspeech/models/contextnet_lstm/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/openspeech/models/contextnet_lstm/configurations.py b/openspeech/models/contextnet_lstm/configurations.py
new file mode 100644
index 0000000..e1de04c
--- /dev/null
+++ b/openspeech/models/contextnet_lstm/configurations.py
@@ -0,0 +1,103 @@
+# MIT License
+#
+# Copyright (c) 2021 Soohwan Kim and Sangchun Ha and Soyoung Cho
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from dataclasses import dataclass, field
+
+from openspeech.dataclass.configurations import OpenspeechDataclass
+
+
+@dataclass
+class ContextNetLSTMConfigs(OpenspeechDataclass):
+    r"""
+    This is the configuration class to store the configuration of
+    a :class:`~openspeech.models.ContextNetLSTM`.
+
+    It is used to initiated an `ContextNetLSTM` model.
+
+    Configuration objects inherit from :class: `~openspeech.dataclass.configs.OpenspeechDataclass`.
+
+    Args:
+        model_name (str): Model name (default: contextnet_lstm)
+        model_size (str, optional): Size of the model['small', 'medium', 'large'] (default : 'medium')
+        input_dim (int, optional): Dimension of input vector (default : 80)
+        num_encoder_layers (int, optional): The number of convolution layers (default : 5)
+        num_decoder_layers (int): The number of decoder layers. (default: 2)
+        kernel_size (int, optional): Value of convolution kernel size (default : 5)
+        num_channels (int, optional): The number of channels in the convolution filter (default: 256)
+        encoder_dim (int, optional): Dimension of encoder output vector (default: 640)
+        num_attention_heads (int): The number of attention heads. (default: 8)
+        attention_dropout_p (float): The dropout probability of attention module. (default: 0.1)
+        decoder_dropout_p (float): The dropout probability of decoder. (default: 0.1)
+        max_length (int): Max decoding length. (default: 128)
+        teacher_forcing_ratio (float): The ratio of teacher forcing. (default: 1.0)
+        rnn_type (str): Type of rnn cell (rnn, lstm, gru) (default: lstm)
+        decoder_attn_mechanism (str): The attention mechanism for decoder. (default: loc)
+        optimizer (str): Optimizer for training. (default: adam)
+    """
+    model_name: str = field(
+        default="contextnet_lstm", metadata={"help": "Model name"}
+    )
+    model_size: str = field(
+        default="medium", metadata={"help": "Model size"}
+    )
+    input_dim: int = field(
+        default=80, metadata={"help": "Dimension of input vector"}
+    )
+    num_encoder_layers: int = field(
+        default=5, metadata={"help": "The number of convolution layers"}
+    )
+    num_decoder_layers: int = field(
+        default=2, metadata={"help": "The number of decoder layers."}
+    )
+    kernel_size: int = field(
+        default=5, metadata={"help": "Value of convolution kernel size"}
+    )
+    num_channels: int = field(
+        default=256, metadata={"help": "The number of channels in the convolution filter"}
+    )
+    encoder_dim: int = field(
+        default=640, metadata={"help": "Dimension of encoder output vector"}
+    )
+    num_attention_heads: int = field(
+        default=8, metadata={"help": "The number of attention heads."}
+    )
+    attention_dropout_p: float = field(
+        default=0.1, metadata={"help": "The dropout probability of attention module."}
+    )
+    decoder_dropout_p: float = field(
+        default=0.1, metadata={"help": "The dropout probability of decoder."}
+    )
+    max_length: int = field(
+        default=128, metadata={"help": "Max decoding length."}
+    )
+    teacher_forcing_ratio: float = field(
+        default=1.0, metadata={"help": "The ratio of teacher forcing. "}
+    )
+    rnn_type: str = field(
+        default="lstm", metadata={"help": "Type of rnn cell (rnn, lstm, gru)"}
+    )
+    decoder_attn_mechanism: str = field(
+        default="loc", metadata={"help": "The attention mechanism for decoder."}
+    )
+    optimizer: str = field(
+        default="adam", metadata={"help": "Optimizer for training."}
+    )
\ No newline at end of file
diff --git a/openspeech/models/contextnet_lstm/model.py b/openspeech/models/contextnet_lstm/model.py
new file mode 100644
index 0000000..9e26339
--- /dev/null
+++ b/openspeech/models/contextnet_lstm/model.py
@@ -0,0 +1,142 @@
+# MIT License
+#
+# Copyright (c) 2021 Soohwan Kim and Sangchun Ha and Soyoung Cho
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from torch import Tensor
+from omegaconf import DictConfig
+from typing import Dict
+from collections import OrderedDict
+
+from openspeech.models import register_model
+from openspeech.models.contextnet_lstm.configurations import ContextNetLSTMConfigs
+from openspeech.encoders import ContextNetEncoder
+from openspeech.models import OpenspeechEncoderDecoderModel
+from openspeech.decoders import LSTMAttentionDecoder
+from openspeech.vocabs.vocab import Vocabulary
+
+
+@register_model('contextnet_lstm', dataclass=ContextNetLSTMConfigs)
+class ContextNetLSTMModel(OpenspeechEncoderDecoderModel):
+    r"""
+    ContextNet encoder + LSTM decoder.
+
+    Args:
+        configs (DictConfig): configuraion set
+        vocab (Vocabulary): vocab of training data
+
+    Inputs:
+        inputs (torch.FloatTensor): A input sequence passed to encoders. Typically for inputs this will be a padded
+            `FloatTensor` of size ``(batch, seq_length, dimension)``.
+        input_lengths (torch.LongTensor): The length of input tensor. ``(batch)``
+
+    Returns:
+        * y_hats (torch.FloatTensor): Result of model predictions.
+    """
+
+    def __init__(self, configs: DictConfig, vocab: Vocabulary, ) -> None:
+        super(ContextNetLSTMModel, self).__init__(configs, vocab)
+
+    def build_model(self):
+        self.encoder = ContextNetEncoder(
+            num_classes=self.num_classes,
+            model_size=self.configs.model.model_size,
+            input_dim=self.configs.audio.num_mels,
+            num_layers=self.configs.model.num_encoder_layers,
+            kernel_size=self.configs.model.kernel_size,
+            num_channels=self.configs.model.num_channels,
+            output_dim=self.configs.model.encoder_dim,
+            joint_ctc_attention=False,
+        )
+        self.decoder = LSTMAttentionDecoder(
+            num_classes=self.num_classes,
+            max_length=self.configs.model.max_length,
+            hidden_state_dim=self.configs.model.encoder_dim,
+            pad_id=self.vocab.pad_id,
+            sos_id=self.vocab.sos_id,
+            eos_id=self.vocab.eos_id,
+            num_heads=self.configs.model.num_attention_heads,
+            dropout_p=self.configs.model.decoder_dropout_p,
+            num_layers=self.configs.model.num_decoder_layers,
+            attn_mechanism=self.configs.model.decoder_attn_mechanism,
+            rnn_type=self.configs.model.rnn_type,
+        )
+
+    def set_beam_decoder(self, beam_size: int = 3):
+        """ Setting beam search decoder """
+        from openspeech.search import BeamSearchLSTM
+        self.decoder = BeamSearchLSTM(
+            decoder=self.decoder,
+            beam_size=beam_size,
+        )
+
+    def forward(self, inputs: Tensor, input_lengths: Tensor) -> Dict[str, Tensor]:
+        r"""
+        Forward propagate a `inputs` and `targets` pair for inference.
+
+        Inputs:
+            inputs (torch.FloatTensor): A input sequence passed to encoders. Typically for inputs this will be a padded
+                `FloatTensor` of size ``(batch, seq_length, dimension)``.
+            input_lengths (torch.LongTensor): The length of input tensor. ``(batch)``
+
+        Returns:
+            * dict (dict): Result of model predictions that contains `y_hats`, `logits`,
+                `encoder_outputs`, `encoder_logits`, `encoder_output_lengths`.
+        """
+        return super(ContextNetLSTMModel, self).forward(inputs, input_lengths)
+
+    def training_step(self, batch: tuple, batch_idx: int) -> OrderedDict:
+        r"""
+        Forward propagate a `inputs` and `targets` pair for training.
+
+        Inputs:
+            batch (tuple): A train batch contains `inputs`, `targets`, `input_lengths`, `target_lengths`
+            batch_idx (int): The index of batch
+
+        Returns:
+            loss (torch.Tensor): loss for training
+        """
+        return super(ContextNetLSTMModel, self).training_step(batch, batch_idx)
+
+    def validation_step(self, batch: tuple, batch_idx: int) -> OrderedDict:
+        r"""
+        Forward propagate a `inputs` and `targets` pair for validation.
+
+        Inputs:
+            batch (tuple): A train batch contains `inputs`, `targets`, `input_lengths`, `target_lengths`
+            batch_idx (int): The index of batch
+
+        Returns:
+            loss (torch.Tensor): loss for training
+        """
+        return super(ContextNetLSTMModel, self).validation_step(batch, batch_idx)
+
+    def test_step(self, batch: tuple, batch_idx: int) -> OrderedDict:
+        r"""
+        Forward propagate a `inputs` and `targets` pair for test.
+
+        Inputs:
+            batch (tuple): A train batch contains `inputs`, `targets`, `input_lengths`, `target_lengths`
+            batch_idx (int): The index of batch
+
+        Returns:
+            loss (torch.Tensor): loss for training
+        """
+        return super(ContextNetLSTMModel, self).test_step(batch, batch_idx)
\ No newline at end of file
diff --git a/openspeech/models/contextnet_transducer/__init__.py b/openspeech/models/contextnet_transducer/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/openspeech/models/contextnet_transducer/configurations.py b/openspeech/models/contextnet_transducer/configurations.py
new file mode 100644
index 0000000..8f955d7
--- /dev/null
+++ b/openspeech/models/contextnet_transducer/configurations.py
@@ -0,0 +1,91 @@
+# MIT License
+#
+# Copyright (c) 2021 Soohwan Kim and Sangchun Ha and Soyoung Cho
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from dataclasses import dataclass, field
+
+from openspeech.dataclass.configurations import OpenspeechDataclass
+
+
+@dataclass
+class ContextNetTransducerConfigs(OpenspeechDataclass):
+    r"""
+    This is the configuration class to store the configuration of
+    a :class:`~openspeech.models.ContextNetTransducer`.
+
+    It is used to initiated an `ContextNetTransducer` model.
+
+    Configuration objects inherit from :class: `~openspeech.dataclass.configs.OpenspeechDataclass`.
+
+    Args:
+        model_name (str): Model name (default: contextnet_transducer)
+        model_size (str, optional): Size of the model['small', 'medium', 'large'] (default : 'medium')
+        input_dim (int, optional): Dimension of input vector (default : 80)
+        num_encoder_layers (int, optional): The number of convolution layers (default : 5)
+        num_decoder_layers (int, optional): The number of rnn layers (default : 1)
+        kernel_size (int, optional): Value of convolution kernel size (default : 5)
+        num_channels (int, optional): The number of channels in the convolution filter (default: 256)
+        hidden_dim (int, optional): The number of features in the decoder hidden state (default : 2048)
+        encoder_dim (int, optional): Dimension of encoder output vector (default: 640)
+        decoder_output_dim (int, optional): Dimension of decoder output vector (default: 640)
+        dropout (float, optional): Dropout probability of decoder (default: 0.1)
+        rnn_type (str, optional): Type of rnn cell (rnn, lstm, gru) (default: lstm)
+        optimizer (str): Optimizer for training. (default: adam)
+    """
+    model_name: str = field(
+        default="contextnet_transducer", metadata={"help": "Model name"}
+    )
+    model_size: str = field(
+        default="medium", metadata={"help": "Model size"}
+    )
+    input_dim: int = field(
+        default=80, metadata={"help": "Dimension of input vector"}
+    )
+    num_encoder_layers: int = field(
+        default=5, metadata={"help": "The number of convolution layers"}
+    )
+    num_decoder_layers: int = field(
+        default=1, metadata={"help": "The number of rnn layers"}
+    )
+    kernel_size: int = field(
+        default=5, metadata={"help": "Value of convolution kernel size"}
+    )
+    num_channels: int = field(
+        default=256, metadata={"help": "The number of channels in the convolution filter"}
+    )
+    hidden_dim: int = field(
+        default=2048, metadata={"help": "The number of features in the decoder hidden state"}
+    )
+    encoder_dim: int = field(
+        default=640, metadata={"help": "Dimension of encoder output vector"}
+    )
+    decoder_output_dim: int = field(
+        default=640, metadata={"help": "Dimension of decoder output vector"}
+    )
+    dropout: float = field(
+        default=0.1, metadata={"help": "Dropout probability of decoder"}
+    )
+    rnn_type: str = field(
+        default='lstm', metadata={"help": "Type of rnn cell"}
+    )
+    optimizer: str = field(
+        default="adam", metadata={"help": "Optimizer for training"}
+    )
\ No newline at end of file
diff --git a/openspeech/models/contextnet_transducer/model.py b/openspeech/models/contextnet_transducer/model.py
new file mode 100644
index 0000000..94c52b1
--- /dev/null
+++ b/openspeech/models/contextnet_transducer/model.py
@@ -0,0 +1,130 @@
+# MIT License
+#
+# Copyright (c) 2021 Soohwan Kim and Sangchun Ha and Soyoung Cho
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from omegaconf import DictConfig
+from torch import Tensor
+from typing import Dict
+from collections import OrderedDict
+
+from openspeech.models import register_model, OpenspeechTransducerModel
+from openspeech.models.contextnet.model import ContextNetEncoder
+from openspeech.models.contextnet_transducer.configurations import ContextNetTransducerConfigs
+from openspeech.models.rnn_transducer.model import RNNTransducerDecoder
+from openspeech.vocabs.vocab import Vocabulary
+
+
+@register_model('contextnet_transducer', dataclass=ContextNetTransducerConfigs)
+class ContextNetTransducerModel(OpenspeechTransducerModel):
+    r"""
+    ContextNet: Improving Convolutional Neural Networks for Automatic Speech Recognition with Global Context
+    Paper: https://arxiv.org/abs/2005.03191
+
+    Args:
+        configs (DictConfig): configuraion set
+        vocab (Vocabulary): vocab of training data
+
+    Inputs:
+        inputs (torch.FloatTensor): A input sequence passed to encoders. Typically for inputs this will be a padded
+            `FloatTensor` of size ``(batch, seq_length, dimension)``.
+        input_lengths (torch.LongTensor): The length of input tensor. ``(batch)``
+
+    Returns:
+        * outputs (dict): Result of model predictions.
+    """
+
+    def __init__(self, configs: DictConfig, vocab: Vocabulary, ) -> None:
+        super(ContextNetTransducerModel, self).__init__(configs, vocab)
+
+    def build_model(self):
+        self.encoder = ContextNetEncoder(
+            num_classes=self.num_classes,
+            model_size=self.configs.model.model_size,
+            input_dim=self.configs.audio.num_mels,
+            num_layers=self.configs.model.num_encoder_layers,
+            kernel_size=self.configs.model.kernel_size,
+            num_channels=self.configs.model.num_channels,
+            output_dim=self.configs.model.encoder_dim,
+            joint_ctc_attention=False,
+        )
+        self.decoder = RNNTransducerDecoder(
+            num_classes=self.num_classes,
+            hidden_state_dim=self.configs.model.decoder_hidden_state_dim,
+            output_dim=self.configs.model.decoder_output_dim,
+            num_layers=self.configs.model.num_decoder_layers,
+            rnn_type=self.configs.model.rnn_type,
+            sos_id=self.vocab.sos_id,
+            eos_id=self.vocab.eos_id,
+            dropout_p=self.configs.model.decoder_dropout_p,
+        )
+
+    def forward(self, inputs: Tensor, input_lengths: Tensor) -> Dict[str, Tensor]:
+        r"""
+        Forward propagate a `inputs` and `targets` pair for inference.
+
+        Inputs:
+            inputs (torch.FloatTensor): A input sequence passed to encoders. Typically for inputs this will be a padded
+                `FloatTensor` of size ``(batch, seq_length, dimension)``.
+            input_lengths (torch.LongTensor): The length of input tensor. ``(batch)``
+
+        Returns:
+            * outputs (dict): Result of model predictions.
+        """
+        return super(ContextNetTransducerModel, self).forward(inputs, input_lengths)
+
+    def training_step(self, batch: tuple, batch_idx: int) -> OrderedDict:
+        r"""
+        Forward propagate a `inputs` and `targets` pair for training.
+
+        Inputs:
+            batch (tuple): A train batch contains `inputs`, `targets`, `input_lengths`, `target_lengths`
+            batch_idx (int): The index of batch
+
+        Returns:
+            loss (torch.Tensor): loss for training
+        """
+        return super(ContextNetTransducerModel, self).training_step(batch, batch_idx)
+
+    def validation_step(self, batch: tuple, batch_idx: int) -> OrderedDict:
+        r"""
+        Forward propagate a `inputs` and `targets` pair for validation.
+
+        Inputs:
+            batch (tuple): A train batch contains `inputs`, `targets`, `input_lengths`, `target_lengths`
+            batch_idx (int): The index of batch
+
+        Returns:
+            loss (torch.Tensor): loss for training
+        """
+        return super(ContextNetTransducerModel, self).validation_step(batch, batch_idx)
+
+    def test_step(self, batch: tuple, batch_idx: int) -> OrderedDict:
+        r"""
+        Forward propagate a `inputs` and `targets` pair for test.
+
+        Inputs:
+            batch (tuple): A train batch contains `inputs`, `targets`, `input_lengths`, `target_lengths`
+            batch_idx (int): The index of batch
+
+        Returns:
+            loss (torch.Tensor): loss for training
+        """
+        return super(ContextNetTransducerModel, self).test_step(batch, batch_idx)
\ No newline at end of file
diff --git a/openspeech/models/openspeech_transducer_model.py b/openspeech/models/openspeech_transducer_model.py
index 3abec9b..594946c 100644
--- a/openspeech/models/openspeech_transducer_model.py
+++ b/openspeech/models/openspeech_transducer_model.py
@@ -252,7 +252,7 @@ def training_step(self, batch: tuple, batch_idx: int) -> OrderedDict:
         return self.collect_outputs(
             'train',
             logits=logits,
-            input_lengths=input_lengths,
+            input_lengths=output_lengths,
             targets=targets,
             target_lengths=target_lengths,
         )
@@ -287,7 +287,7 @@ def validation_step(self, batch: tuple, batch_idx: int) -> OrderedDict:
         return self.collect_outputs(
             'val',
             logits=None,
-            input_lengths=input_lengths,
+            input_lengths=output_lengths,
             targets=targets,
             target_lengths=target_lengths,
             predictions=predictions,
@@ -323,7 +323,7 @@ def test_step(self, batch: tuple, batch_idx: int) -> OrderedDict:
         return self.collect_outputs(
             'test',
             logits=None,
-            input_lengths=input_lengths,
+            input_lengths=output_lengths,
             targets=targets,
             target_lengths=target_lengths,
             predictions=predictions,
diff --git a/openspeech/modules/contextnet_block.py b/openspeech/modules/contextnet_block.py
new file mode 100644
index 0000000..934e03b
--- /dev/null
+++ b/openspeech/modules/contextnet_block.py
@@ -0,0 +1,199 @@
+# MIT License
+#
+# Copyright (c) 2021 Soohwan Kim and Sangchun Ha and Soyoung Cho
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from typing import Tuple
+from torch import Tensor
+from openspeech.modules.contextnet_module import ContextNetConvModule, ContextNetSEModule
+from openspeech.modules.swish import Swish
+import torch.nn as nn
+
+
+class ContextNetBlock(nn.Module):
+    r"""
+    Convolution block contains a number of convolutions, each followed by batch normalization and activation.
+    Squeeze-and-excitation (SE) block operates on the output of the last convolution layer.
+    Skip connection with projection is applied on the output of the squeeze-and-excitation block.
+
+    Args:
+        in_channels (int): Input channel in convolutional layer
+        out_channels (int): Output channel in convolutional layer
+        num_layers (int, optional): The number of convolutional layers (default : 5)
+        kernel_size (int, optional): Value of convolution kernel size (default : 5)
+        stride(int, optional): Value of stride (default : 1)
+        padding (int, optional): Value of padding (default: 0)
+        residual (bool, optional): Flag indication residual or not (default : True)
+
+    Inputs: inputs, input_lengths
+        - **inputs**: Input of convolution block `FloatTensor` of size ``(batch, dimension, seq_length)``
+        - **input_lengths**: The length of input tensor. ``(batch)``
+
+    Returns: output, output_lengths
+        - **output**: Output of convolution block `FloatTensor` of size
+                ``(batch, dimension, seq_length)``
+        - **output_lengths**: The length of output tensor. ``(batch)``
+    """
+    def __init__(
+            self,
+            in_channels: int,
+            out_channels: int,
+            num_layers: int = 5,
+            kernel_size: int = 5,
+            stride: int = 1,
+            padding: int = 0,
+            residual: bool = True,
+    ) -> None:
+        super(ContextNetBlock, self).__init__()
+        self.num_layers = num_layers
+        self.swish = Swish()
+        self.se_layer = ContextNetSEModule(out_channels)
+        self.residual = None
+
+        if residual:
+            self.residual = ContextNetConvModule(
+                in_channels=in_channels,
+                out_channels=out_channels,
+                kernel_size=kernel_size,
+                stride=stride,
+                padding=padding,
+                activation=False,
+            )
+
+        if self.num_layers == 1:
+            self.conv_layers = ContextNetConvModule(
+                        in_channels=in_channels,
+                        out_channels=out_channels,
+                        kernel_size=kernel_size,
+                        stride=stride,
+                        padding=padding,
+                    )
+
+        else:
+            stride_list = [1 for _ in range(num_layers - 1)] + [stride]
+            in_channel_list = [in_channels] + [out_channels for _ in range(num_layers - 1)]
+
+            self.conv_layers = nn.ModuleList(list())
+            for in_channels, stride in zip(in_channel_list, stride_list):
+                self.conv_layers.append(
+                    ContextNetConvModule(
+                        in_channels=in_channels,
+                        out_channels=out_channels,
+                        kernel_size=kernel_size,
+                        stride=stride,
+                        padding=padding,
+                    )
+                )
+
+    def forward(
+            self,
+            inputs: Tensor,
+            input_lengths: Tensor,
+    ) -> Tuple[Tensor, Tensor]:
+        r"""
+        Forward propagate a `inputs` for convolution block.
+
+        Args:
+            **inputs** (torch.FloatTensor): Input of convolution block `FloatTensor` of size
+                ``(batch, dimension, seq_length)``
+            **input_lengths** (torch.LongTensor): The length of input tensor. ``(batch)``
+
+        Returns:
+            **output** (torch.FloatTensor): Output of convolution block `FloatTensor` of size
+                ``(batch, dimension, seq_length)``
+            **output_lengths** (torch.LongTensor): The length of output tensor. ``(batch)``
+        """
+        output = inputs
+        output_lengths = input_lengths
+
+        if self.num_layers == 1:
+            output, output_lengths = self.conv_layers(output, output_lengths)
+        else:
+            for conv_layer in self.conv_layers:
+                output, output_lengths = conv_layer(output, output_lengths)
+
+        output = self.se_layer(output, output_lengths)
+
+        if self.residual is not None:
+            residual, _ = self.residual(inputs, input_lengths)
+            output += residual
+
+        return self.swish(output), output_lengths
+
+    @staticmethod
+    def make_conv_blocks(
+            input_dim: int = 80,
+            num_layers: int = 5,
+            kernel_size: int = 5,
+            num_channels: int = 256,
+            output_dim: int = 640,
+    ) -> nn.ModuleList:
+        r"""
+        Create 23 convolution blocks.
+
+        Args:
+            input_dim (int, optional): Dimension of input vector (default : 80)
+            num_layers (int, optional): The number of convolutional layers (default : 5)
+            kernel_size (int, optional): Value of convolution kernel size (default : 5)
+            num_channels (int, optional): The number of channels in the convolution filter (default: 256)
+            output_dim (int, optional): Dimension of encoder output vector (default: 640)
+
+        Returns:
+            **conv_blocks** (nn.ModuleList): ModuleList with 23 convolution blocks
+        """
+        conv_blocks = nn.ModuleList()
+
+        # C0 : 1 conv layer, init_dim output channels, stride 1, no residual
+        conv_blocks.append(ContextNetBlock(input_dim, num_channels, 1, kernel_size, 1, 0, False))
+
+        # C1-2 : 5 conv layers, init_dim output channels, stride 1
+        for _ in range(1, 2 + 1):
+            conv_blocks.append(ContextNetBlock(num_channels, num_channels, num_layers, kernel_size, 1, 0, True))
+
+        # C3 : 5 conv layer, init_dim output channels, stride 2
+        conv_blocks.append(ContextNetBlock(num_channels, num_channels, num_layers, kernel_size, 2, 0, True))
+
+        # C4-6 : 5 conv layers, init_dim output channels, stride 1
+        for _ in range(4, 6 + 1):
+            conv_blocks.append(ContextNetBlock(num_channels, num_channels, num_layers, kernel_size, 1, 0, True))
+
+        # C7 : 5 conv layers, init_dim output channels, stride 2
+        conv_blocks.append(ContextNetBlock(num_channels, num_channels, num_layers, kernel_size, 2, 0, True))
+
+        # C8-10 : 5 conv layers, init_dim output channels, stride 1
+        for _ in range(8, 10 + 1):
+            conv_blocks.append(ContextNetBlock(num_channels, num_channels, num_layers, kernel_size, 1, 0, True))
+
+        # C11-13 : 5 conv layers, middle_dim output channels, stride 1
+        conv_blocks.append(ContextNetBlock(num_channels, num_channels << 1, num_layers, kernel_size, 1, 0, True))
+        for _ in range(12, 13 + 1):
+            conv_blocks.append(ContextNetBlock(num_channels << 1, num_channels << 1, num_layers, kernel_size, 1, 0, True))
+
+        # C14 : 5 conv layers, middle_dim output channels, stride 2
+        conv_blocks.append(ContextNetBlock(num_channels << 1, num_channels << 1, num_layers, kernel_size, 2, 0, True))
+
+        # C15-21 : 5 conv layers, middle_dim output channels, stride 1
+        for i in range(15, 21 + 1):
+            conv_blocks.append(ContextNetBlock(num_channels << 1, num_channels << 1, num_layers, kernel_size, 1, 0, True))
+
+        # C22 : 1 conv layer, final_dim output channels, stride 1, no residual
+        conv_blocks.append(ContextNetBlock(num_channels << 1, output_dim, 1, kernel_size, 1, 0, False))
+
+        return conv_blocks
\ No newline at end of file
diff --git a/openspeech/modules/contextnet_module.py b/openspeech/modules/contextnet_module.py
new file mode 100644
index 0000000..0540dbb
--- /dev/null
+++ b/openspeech/modules/contextnet_module.py
@@ -0,0 +1,182 @@
+# MIT License
+#
+# Copyright (c) 2021 Soohwan Kim and Sangchun Ha and Soyoung Cho
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from torch import Tensor
+from typing import Tuple
+from openspeech.modules.swish import Swish
+import torch.nn as nn
+
+
+class ContextNetSEModule(nn.Module):
+    r"""
+    Squeeze-and-excitation module.
+
+    Args:
+        dim (int): Dimension to be used for two fully connected (FC) layers
+
+    Inputs: inputs, input_lengths
+        - **inputs**: The output of the last convolution layer. `FloatTensor` of size
+            ``(batch, dimension, seq_length)``
+        - **input_lengths**: The length of input tensor. ``(batch)``
+
+    Returns: output
+        - **output**: Output of SELayer `FloatTensor` of size
+            ``(batch, dimension, seq_length)``
+    """
+    def __init__(self, dim: int) -> None:
+        super(ContextNetSEModule, self).__init__()
+        assert dim % 8 == 0, 'Dimension should be divisible by 8.'
+
+        self.dim = dim
+        self.sequential = nn.Sequential(
+            nn.Linear(dim, dim // 8),
+            Swish(),
+            nn.Linear(dim // 8, dim),
+        )
+
+    def forward(
+            self,
+            inputs: Tensor,
+            input_lengths: Tensor,
+    ) -> Tuple[Tensor, Tensor]:
+        r"""
+        Forward propagate a `inputs` for SE Layer.
+
+        Args:
+            **inputs** (torch.FloatTensor): The output of the last convolution layer. `FloatTensor` of size
+                ``(batch, dimension, seq_length)``
+            **input_lengths** (torch.LongTensor): The length of input tensor. ``(batch)``
+
+        Returns:
+            **output** (torch.FloatTensor): Output of SELayer `FloatTensor` of size
+                ``(batch, dimension, seq_length)``
+        """
+        residual = inputs
+        seq_lengths = inputs.size(2)
+
+        inputs = inputs.sum(dim=2) / input_lengths.unsqueeze(1)
+        output = self.sequential(inputs)
+
+        output = output.sigmoid().unsqueeze(2)
+        output = output.repeat(1, 1, seq_lengths)
+
+        return output * residual
+
+
+class ContextNetConvModule(nn.Module):
+    r"""
+    When the stride is 1, it pads the input so the output has the shape as the input.
+    And when the stride is 2, it does not pad the input.
+
+    Args:
+        in_channels (int): Input channel in convolutional layer
+        out_channels (int): Output channel in convolutional layer
+        kernel_size (int, optional): Value of convolution kernel size (default : 5)
+        stride(int, optional): Value of stride (default : 1)
+        padding (int, optional): Value of padding (default: 0)
+        activation (bool, optional): Flag indication use activation function or not (default : True)
+        groups(int, optional): Value of groups (default : 1)
+        bias (bool, optional): Flag indication use bias or not (default : True)
+
+    Inputs: inputs, input_lengths
+        - **inputs**: Input of convolution layer `FloatTensor` of size ``(batch, dimension, seq_length)``
+        - **input_lengths**: The length of input tensor. ``(batch)``
+
+    Returns: output, output_lengths
+        - **output**: Output of convolution layer `FloatTensor` of size
+                ``(batch, dimension, seq_length)``
+        - **output_lengths**: The length of output tensor. ``(batch)``
+    """
+    def __init__(
+            self,
+            in_channels: int,
+            out_channels: int,
+            kernel_size: int = 5,
+            stride: int = 1,
+            padding: int = 0,
+            activation: bool = True,
+            groups: int = 1,
+            bias: bool = True,
+    ):
+        super(ContextNetConvModule, self).__init__()
+        assert kernel_size == 5, "The convolution layer in the ContextNet model has 5 kernels."
+
+        if stride == 1:
+            self.conv = nn.Conv1d(
+                in_channels=in_channels,
+                out_channels=out_channels,
+                kernel_size=kernel_size,
+                stride=stride,
+                dilation=1,
+                padding=(kernel_size - 1) // 2,
+                groups=groups,
+                bias=bias,
+            )
+        elif stride == 2:
+            self.conv = nn.Conv1d(
+                in_channels=in_channels,
+                out_channels=out_channels,
+                kernel_size=kernel_size,
+                stride=stride,
+                dilation=1,
+                padding=padding,
+                groups=groups,
+                bias=bias,
+            )
+
+        self.batch_norm = nn.BatchNorm1d(num_features=out_channels)
+        self.activation = activation
+
+        if self.activation:
+            self.swish = Swish()
+
+    def forward(
+            self,
+            inputs: Tensor,
+            input_lengths: Tensor,
+    ) -> Tuple[Tensor, Tensor]:
+        r"""
+        Forward propagate a `inputs` for convolution layer.
+
+        Args:
+            **inputs** (torch.FloatTensor): Input of convolution layer `FloatTensor` of size
+                ``(batch, dimension, seq_length)``
+            **input_lengths** (torch.LongTensor): The length of input tensor. ``(batch)``
+
+        Returns:
+            **output** (torch.FloatTensor): Output of convolution layer `FloatTensor` of size
+                ``(batch, dimension, seq_length)``
+            **output_lengths** (torch.LongTensor): The length of output tensor. ``(batch)``
+        """
+        outputs, output_lengths = self.conv(inputs), self._get_sequence_lengths(input_lengths)
+        outputs = self.batch_norm(outputs)
+
+        if self.activation:
+            outputs = self.swish(outputs)
+
+        return outputs, output_lengths
+
+    def _get_sequence_lengths(self, seq_lengths):
+        return (
+                (seq_lengths + 2 * self.conv.padding[0]
+                 - self.conv.dilation[0] * (self.conv.kernel_size[0] - 1) - 1) // self.conv.stride[0] + 1
+        )


Test patch:
diff --git a/tests/test_conformer_transducer.py b/tests/test_conformer_transducer.py
index 490eed7..fb3655c 100644
--- a/tests/test_conformer_transducer.py
+++ b/tests/test_conformer_transducer.py
@@ -35,7 +35,7 @@ def test_forward(self):
             loss = rnnt_loss(
                 outputs["logits"],
                 DUMMY_TARGETS,
-                DUMMY_INPUT_LENGTHS,
+                outputs["encoder_output_lengths"],
                 DUMMY_TARGET_LENGTHS,
                 reduction="mean",
                 blank=vocab.blank_id,
diff --git a/tests/test_contextnet.py b/tests/test_contextnet.py
new file mode 100644
index 0000000..16b7545
--- /dev/null
+++ b/tests/test_contextnet.py
@@ -0,0 +1,79 @@
+import unittest
+import torch
+import torch.nn as nn
+import logging
+
+from openspeech.criterion.ctc.ctc import CTCLossConfigs
+from openspeech.models import ContextNetConfigs, ContextNetModel
+from openspeech.utils import DUMMY_INPUTS, DUMMY_INPUT_LENGTHS, DUMMY_TARGETS, DUMMY_TARGET_LENGTHS, build_dummy_configs
+from openspeech.vocabs.ksponspeech.character import KsponSpeechCharacterVocabulary
+
+logger = logging.getLogger(__name__)
+
+
+class TestConformer(unittest.TestCase):
+    def test_forward(self):
+        configs = build_dummy_configs(model_configs=ContextNetConfigs(), criterion_configs=CTCLossConfigs())
+
+        vocab = KsponSpeechCharacterVocabulary(configs)
+        model = ContextNetModel(configs, vocab)
+        model.build_model()
+
+        criterion = nn.CTCLoss(blank=3, reduction='mean', zero_infinity=True)
+        optimizer = torch.optim.Adam(model.parameters(), lr=1e-04)
+
+        for i in range(10):
+            outputs = model(DUMMY_INPUTS, DUMMY_INPUT_LENGTHS)
+
+            loss = criterion(
+                outputs["logits"].transpose(0, 1),
+                DUMMY_TARGETS[:, 1:],
+                outputs["output_lengths"],
+                DUMMY_TARGET_LENGTHS,
+            )
+            loss.backward()
+            optimizer.step()
+            assert type(loss.item()) == float
+
+    def test_training_step(self):
+        configs = build_dummy_configs(model_configs=ContextNetConfigs(), criterion_configs=CTCLossConfigs())
+
+        vocab = KsponSpeechCharacterVocabulary(configs)
+        model = ContextNetModel(configs, vocab)
+        model.build_model()
+
+        for i in range(5):
+            outputs = model.training_step(
+                batch=(DUMMY_INPUTS, DUMMY_TARGETS, DUMMY_INPUT_LENGTHS, DUMMY_TARGET_LENGTHS), batch_idx=i
+            )
+            assert type(outputs["loss"].item()) == float
+
+    def test_validation_step(self):
+        configs = build_dummy_configs(model_configs=ContextNetConfigs(), criterion_configs=CTCLossConfigs())
+
+        vocab = KsponSpeechCharacterVocabulary(configs)
+        model = ContextNetModel(configs, vocab)
+        model.build_model()
+
+        for i in range(5):
+            outputs = model.validation_step(
+                batch=(DUMMY_INPUTS, DUMMY_TARGETS, DUMMY_INPUT_LENGTHS, DUMMY_TARGET_LENGTHS), batch_idx=i
+            )
+            assert type(outputs["loss"].item()) == float
+
+    def test_test_step(self):
+        configs = build_dummy_configs(model_configs=ContextNetConfigs(), criterion_configs=CTCLossConfigs())
+
+        vocab = KsponSpeechCharacterVocabulary(configs)
+        model = ContextNetModel(configs, vocab)
+        model.build_model()
+
+        for i in range(5):
+            outputs = model.test_step(
+                batch=(DUMMY_INPUTS, DUMMY_TARGETS, DUMMY_INPUT_LENGTHS, DUMMY_TARGET_LENGTHS), batch_idx=i
+            )
+            assert type(outputs["loss"].item()) == float
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/test_contextnet_lstm.py b/tests/test_contextnet_lstm.py
new file mode 100644
index 0000000..0db3293
--- /dev/null
+++ b/tests/test_contextnet_lstm.py
@@ -0,0 +1,111 @@
+import unittest
+import torch
+import logging
+
+from openspeech.models import ContextNetLSTMConfigs, ContextNetLSTMModel
+from openspeech.vocabs.ksponspeech.character import KsponSpeechCharacterVocabulary
+from openspeech.utils import (
+    DUMMY_INPUTS,
+    DUMMY_INPUT_LENGTHS,
+    DUMMY_TARGETS,
+    DUMMY_TARGET_LENGTHS,
+    build_dummy_configs,
+)
+from openspeech.criterion.label_smoothed_cross_entropy.label_smoothed_cross_entropy import (
+    LabelSmoothedCrossEntropyLossConfigs,
+    LabelSmoothedCrossEntropyLoss,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class TestConformerLSTM(unittest.TestCase):
+    def test_forward(self):
+        configs = build_dummy_configs(
+            model_configs=ContextNetLSTMConfigs(),
+            criterion_configs=LabelSmoothedCrossEntropyLossConfigs(),
+        )
+
+        vocab = KsponSpeechCharacterVocabulary(configs)
+        model = ContextNetLSTMModel(configs, vocab)
+        model.build_model()
+
+        criterion = LabelSmoothedCrossEntropyLoss(configs, num_classes=len(vocab), vocab=vocab)
+        optimizer = torch.optim.Adam(model.parameters(), lr=1e-04)
+
+        for i in range(3):
+            outputs = model(DUMMY_INPUTS, DUMMY_INPUT_LENGTHS)
+
+            loss = criterion(outputs["logits"], DUMMY_TARGETS[:, 1:])
+            loss.backward()
+            optimizer.step()
+            print(loss.item())
+
+            assert type(loss.item()) == float
+
+    def test_beam_search(self):
+        configs = build_dummy_configs(
+            model_configs=ContextNetLSTMConfigs(),
+            criterion_configs=LabelSmoothedCrossEntropyLossConfigs(),
+        )
+
+        vocab = KsponSpeechCharacterVocabulary(configs)
+        model = ContextNetLSTMModel(configs, vocab)
+        model.build_model()
+        model.set_beam_decoder(beam_size=3)
+
+        for i in range(3):
+            prediction = model(DUMMY_INPUTS, DUMMY_INPUT_LENGTHS)["predictions"]
+            assert isinstance(prediction, torch.Tensor)
+
+    def test_training_step(self):
+        configs = build_dummy_configs(
+            model_configs=ContextNetLSTMConfigs(),
+            criterion_configs=LabelSmoothedCrossEntropyLossConfigs(),
+        )
+
+        vocab = KsponSpeechCharacterVocabulary(configs)
+        model = ContextNetLSTMModel(configs, vocab)
+        model.build_model()
+
+        for i in range(3):
+            outputs = model.training_step(
+                batch=(DUMMY_INPUTS, DUMMY_TARGETS, DUMMY_INPUT_LENGTHS, DUMMY_TARGET_LENGTHS), batch_idx=i
+            )
+            assert type(outputs["loss"].item()) == float
+
+    def test_validation_step(self):
+        configs = build_dummy_configs(
+            model_configs=ContextNetLSTMConfigs(),
+            criterion_configs=LabelSmoothedCrossEntropyLossConfigs(),
+        )
+
+        vocab = KsponSpeechCharacterVocabulary(configs)
+        model = ContextNetLSTMModel(configs, vocab)
+        model.build_model()
+
+        for i in range(3):
+            outputs = model.validation_step(
+                batch=(DUMMY_INPUTS, DUMMY_TARGETS, DUMMY_INPUT_LENGTHS, DUMMY_TARGET_LENGTHS), batch_idx=i
+            )
+            assert type(outputs["loss"].item()) == float
+
+    def test_test_step(self):
+        configs = build_dummy_configs(
+            model_configs=ContextNetLSTMConfigs(),
+            criterion_configs=LabelSmoothedCrossEntropyLossConfigs(),
+        )
+
+        vocab = KsponSpeechCharacterVocabulary(configs)
+        model = ContextNetLSTMModel(configs, vocab)
+        model.build_model()
+
+        for i in range(3):
+            outputs = model.test_step(
+                batch=(DUMMY_INPUTS, DUMMY_TARGETS, DUMMY_INPUT_LENGTHS, DUMMY_TARGET_LENGTHS), batch_idx=i
+            )
+            assert type(outputs["loss"].item()) == float
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/test_contextnet_transducer.py b/tests/test_contextnet_transducer.py
new file mode 100644
index 0000000..388b269
--- /dev/null
+++ b/tests/test_contextnet_transducer.py
@@ -0,0 +1,98 @@
+import unittest
+import torch
+import logging
+
+from openspeech.criterion.transducer.transducer import TransducerLossConfigs
+from openspeech.models import ContextNetTransducerModel, ContextNetTransducerConfigs
+from openspeech.utils import DUMMY_INPUTS, DUMMY_INPUT_LENGTHS, DUMMY_TARGETS, DUMMY_TARGET_LENGTHS, \
+    build_dummy_configs, WARPRNNT_IMPORT_ERROR
+from openspeech.vocabs.ksponspeech.character import KsponSpeechCharacterVocabulary
+
+try:
+    from warp_rnnt import rnnt_loss
+except ImportError:
+    raise ImportError(WARPRNNT_IMPORT_ERROR)
+
+logger = logging.getLogger(__name__)
+
+
+class TestConformerTransducer(unittest.TestCase):
+    def test_forward(self):
+        configs = build_dummy_configs(
+            model_configs=ContextNetTransducerConfigs(),
+            criterion_configs=TransducerLossConfigs(),
+        )
+
+        vocab = KsponSpeechCharacterVocabulary(configs)
+        model = ContextNetTransducerModel(configs, vocab)
+        model.build_model()
+
+        optimizer = torch.optim.Adam(model.parameters(), lr=1e-04)
+
+        for i in range(3):
+            outputs = model(DUMMY_INPUTS, DUMMY_INPUT_LENGTHS)
+
+            loss = rnnt_loss(
+                outputs["logits"],
+                DUMMY_TARGETS,
+                outputs["encoder_output_lengths"],
+                DUMMY_TARGET_LENGTHS,
+                reduction="mean",
+                blank=vocab.blank_id,
+                gather=True,
+            )
+            loss.backward()
+            optimizer.step()
+            assert type(loss.item()) == float
+
+    def test_training_step(self):
+        configs = build_dummy_configs(
+            model_configs=ContextNetTransducerConfigs(),
+            criterion_configs=TransducerLossConfigs(),
+        )
+
+        vocab = KsponSpeechCharacterVocabulary(configs)
+        model = ContextNetTransducerModel(configs, vocab)
+        model.build_model()
+
+        for i in range(3):
+            outputs = model.training_step(
+                batch=(DUMMY_INPUTS, DUMMY_TARGETS, DUMMY_INPUT_LENGTHS, DUMMY_TARGET_LENGTHS), batch_idx=i
+            )
+            assert type(outputs["loss"].item()) == float
+
+    def test_validation_step(self):
+        configs = build_dummy_configs(
+            model_configs=ContextNetTransducerConfigs(),
+            criterion_configs=TransducerLossConfigs(),
+        )
+
+        vocab = KsponSpeechCharacterVocabulary(configs)
+        model = ContextNetTransducerModel(configs, vocab)
+        model.build_model()
+
+        for i in range(3):
+            outputs = model.validation_step(
+                batch=(DUMMY_INPUTS, DUMMY_TARGETS, DUMMY_INPUT_LENGTHS, DUMMY_TARGET_LENGTHS), batch_idx=i
+            )
+            assert type(outputs["loss"].item()) == float
+
+    def test_test_step(self):
+        configs = build_dummy_configs(
+            model_configs=ContextNetTransducerConfigs(),
+            criterion_configs=TransducerLossConfigs(),
+        )
+
+        vocab = KsponSpeechCharacterVocabulary(configs)
+        model = ContextNetTransducerModel(configs, vocab)
+        model.build_model()
+
+        for i in range(3):
+            outputs = model.test_step(
+                batch=(DUMMY_INPUTS, DUMMY_TARGETS, DUMMY_INPUT_LENGTHS, DUMMY_TARGET_LENGTHS), batch_idx=i
+            )
+            assert type(outputs["loss"].item()) == float
+
+
+if __name__ == '__main__':
+    unittest.main()