Repository: cisco/mindmeld (language: python)

Issue:
Create a model registry with DVC and DAGsHub (P2)
This PR enables Mindmeld users to create a central model registry on DAGsHub, making the model easily accessible and shareable with other team members by hosting all models in a central hub.

This refers to issue #300, which I discussed with @vijay120 and @alhuang10.

I added the option:
`python -m home_assistant dvc --setup_dagshub`

This defines the DVC remote as DAGsHub and guides the user in creating a project and providing the required information (URL, username, and password/access token).

I also edited the documentation to cover this change, and as far as I could test everything is passing.
Looking forward to getting your input on this :)


Discussion / hints:
@vijay120 gentle reminder. Would appreciate your input on this 🙏

Fix patch:
diff --git a/extras-requirements.txt b/extras-requirements.txt
index 4bde56353..38eebe6cf 100644
--- a/extras-requirements.txt
+++ b/extras-requirements.txt
@@ -4,5 +4,8 @@
 -e .[examples]
 -e .[augment]
 -e .[language_annotator]
+-e .[elasticsearch]
 -e .[active_learning]
--e .[dvc]
\ No newline at end of file
+-e .[dvc]
+-e .[torch]
+-e .[transformers]
diff --git a/mindmeld/augmentation.py b/mindmeld/augmentation.py
index 8e7c97258..5d9cca6a1 100644
--- a/mindmeld/augmentation.py
+++ b/mindmeld/augmentation.py
@@ -34,7 +34,7 @@
     PARAPHRASER_FILE_PATH, \
     PARAPHRASER_MODEL_PATH, \
     HUGGINGFACE_PARAPHRASER_MODEL_PATH
-from .models.taggers.embeddings import TqdmUpTo
+from .models.containers import TqdmUpTo
 
 logger = logging.getLogger(__name__)
 
@@ -472,6 +472,7 @@ def _generate_paraphrases(self, processed_queries):
             batch = self.tokenizer.prepare_seq2seq_batch(
                 tokenizer_input,
                 **self.default_tokenizer_params,
+                return_tensors="pt",
             ).to(self.torch_device)
             with _get_module_or_attr("torch", "no_grad")():
                 generated = self.model.generate(
diff --git a/mindmeld/components/classifier.py b/mindmeld/components/classifier.py
index 8f11b6605..e5fa59263 100644
--- a/mindmeld/components/classifier.py
+++ b/mindmeld/components/classifier.py
@@ -68,15 +68,9 @@ def __init__(
         param_selection=None,
     ):
         """Initializes a classifier configuration"""
-        for arg, val in {"model_type": model_type, "features": features}.items():
+        for arg, val in {"model_type": model_type}.items():
             if val is None:
                 raise TypeError("__init__() missing required argument {!r}".format(arg))
-        if params is None and (
-            param_selection is None or param_selection.get("grid") is None
-        ):
-            raise ValueError(
-                "__init__() One of 'params' and 'param_selection' is required"
-            )
         self.model_type = model_type
         self.features = features
         self.model_settings = model_settings
diff --git a/mindmeld/components/entity_resolver.py b/mindmeld/components/entity_resolver.py
index 8281b555e..4403fb7e1 100644
--- a/mindmeld/components/entity_resolver.py
+++ b/mindmeld/components/entity_resolver.py
@@ -463,6 +463,17 @@ def load(self, path, entity_map=None):
         self.ready = True
         self.dirty = False
 
+    # TODO: method to be removed in a next major release of Mindmeld
+    @abstractmethod
+    def load_deprecated(self):
+        """
+        A method to handle the deprecated way of using the .load() method in entity resolvers. This
+        ensures backwards compatibility when loading models that were built using an older version
+        of Mindmeld i.e a version <=4.4.0. Since no hash pickle file is dumped in the older version
+        of MindMeld, using the latest .load() method throws a FileNotFoundError.
+        """
+        raise NotImplementedError
+
     def unload(self):
         """
         Unloads the model from memory. This helps reduce memory requirements while
@@ -784,6 +795,9 @@ def _load(self, path, entity_map):
     def _unload(self):
         self.processed_entity_map = None
 
+    def load_deprecated(self):
+        self.fit()
+
 
 class ElasticsearchEntityResolver(BaseEntityResolver):
     """
@@ -1221,6 +1235,30 @@ def _load(self, path, entity_map):
         except _getattr("elasticsearch", "ElasticsearchException") as e:
             raise EntityResolverError from e
 
+    def load_deprecated(self):
+        try:
+            scoped_index_name = get_scoped_index_name(
+                self._app_namespace, self._es_index_name
+            )
+            if not self._es_client.indices.exists(index=scoped_index_name):
+                self.fit()
+        except _getattr("elasticsearch", "ConnectionError") as e:
+            logger.error(
+                "Unable to connect to Elasticsearch: %s details: %s", e.error, e.info
+            )
+            raise ElasticsearchConnectionError(es_host=self._es_client.transport.hosts) from e
+        except _getattr("elasticsearch", "TransportError") as e:
+            logger.error(
+                "Unexpected error occurred when sending requests to Elasticsearch: %s "
+                "Status code: %s details: %s",
+                e.error,
+                e.status_code,
+                e.info,
+            )
+            raise EntityResolverError from e
+        except _getattr("elasticsearch", "ElasticsearchException") as e:
+            raise EntityResolverError from e
+
 
 class TfIdfSparseCosSimEntityResolver(BaseEntityResolver):
     # pylint: disable=too-many-instance-attributes
@@ -1526,6 +1564,9 @@ def find_similarity(
 
         return results
 
+    def load_deprecated(self):
+        self.fit()
+
 
 class EmbedderCosSimEntityResolver(BaseEntityResolver):
     """
@@ -1786,6 +1827,9 @@ def predict_batch(self, entity_list, top_n: int = DEFAULT_TOP_N, batch_size: int
 
         return [self._trim_and_sort_results(results, top_n) for results in results_list]
 
+    def load_deprecated(self):
+        self.fit()
+
 
 class SentenceBertCosSimEntityResolver(EmbedderCosSimEntityResolver):
     """
diff --git a/mindmeld/components/nlp.py b/mindmeld/components/nlp.py
index bdfccb400..233fb828f 100644
--- a/mindmeld/components/nlp.py
+++ b/mindmeld/components/nlp.py
@@ -1718,9 +1718,19 @@ def _load(self, incremental_timestamp=None):
             self.type,
             timestamp=incremental_timestamp,
         )
-        self.entity_resolver.load(
-            incremental_model_path if incremental_timestamp else model_path
-        )
+        try:
+            self.entity_resolver.load(
+                incremental_model_path if incremental_timestamp else model_path
+            )
+        except FileNotFoundError as e:
+            logger.error(e)
+            msg = "No cached hash (pkl) file found. This can happen if you are trying to load " \
+                  "entity resolvers that were built using mindmeld version <=4.4.0 but trying to " \
+                  "load them using version >4.4.0"
+            msg += "\nConsider doing an incremental build of your nlp hierarchy to not see this " \
+                   "error message again."
+            logger.error(msg)
+            self.entity_resolver.load_deprecated()
 
     def _evaluate(self, print_stats, label_set="test"):
         # evaluation can be done only for role classifier and not for entity resolver
diff --git a/mindmeld/models/__init__.py b/mindmeld/models/__init__.py
index 5ab4fff9a..43da63cab 100644
--- a/mindmeld/models/__init__.py
+++ b/mindmeld/models/__init__.py
@@ -14,11 +14,10 @@
 # Need to import the following so models, label encoders, features are registered.
 # See helpers module
 from . import (
-    auto_model,
+    model_factory,
     embedder_models,
     labels
 )
-from .auto_model import AutoModel
 from .embedder_models import Embedder
 from .features import (
     entity_features,
@@ -35,12 +34,13 @@
     register_embedder,
 )
 from .model import ModelConfig
+from .model_factory import ModelFactory
 
 __all__ = [
-    "auto_model",
+    "model_factory",
     "embedder_models",
     "labels",
-    "AutoModel",
+    "ModelFactory",
     "Embedder",
     "query_features",
     "entity_features",
diff --git a/mindmeld/models/_util.py b/mindmeld/models/_util.py
index c6915ab70..ba560a1bd 100644
--- a/mindmeld/models/_util.py
+++ b/mindmeld/models/_util.py
@@ -44,7 +44,7 @@ def torch_op(op, *args, sub="", **kwargs):
     """
     A safe function caller for torch module
     Usage:
-        call ```_torch("normalize", arg1, arg2, sub="nn.functional", p=2, dim=1)```
+        call ```torch_op("normalize", arg1, arg2, sub="nn.functional", p=2, dim=1)```
         instead of ```torch.nn.functional.normalize(arg1, arg2, p=2, dim=1)```
     """
 
diff --git a/mindmeld/models/auto_model.py b/mindmeld/models/auto_model.py
deleted file mode 100644
index 6a9bf9a1a..000000000
--- a/mindmeld/models/auto_model.py
+++ /dev/null
@@ -1,141 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (c) 2015 Cisco Systems, Inc. and others.  All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#     http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Contains classes that can resolve model types and create/load appropriate models
-"""
-
-import logging
-from typing import Union, Type
-
-from .helpers import register_model
-from .model import ModelConfig, AbstractModel
-from .tagger_models import AutoTaggerModel
-from .text_models import AutoTextModel
-
-logger = logging.getLogger(__name__)
-
-
-class AutoModel:
-    """Auto class that identifies appropriate text/tagger model from text_models.py/tagger_models.py
-    to load one based on the inputted configs or from the loaded configs file.
-
-    The .from_config() methods allows to load the appropriate model when a ModelConfig is passed.
-    The .from_path() method uses AbstractModel's load method to load a dumped config, which is then
-    used to load appropriate model and return it through a metadata dictionary object.
-
-    """
-    ALLOWED_MODEL_TYPES = ["text", "tagger"]
-
-    def __new__(cls, config: Union[dict, ModelConfig]) -> Type[AbstractModel]:
-        # method for backwards compatability in ./helpers/create_model()
-        return cls.from_config(config)
-
-    @classmethod
-    def from_config(cls, model_config: Union[dict, ModelConfig]) -> Type[AbstractModel]:
-        """
-        Loads a valid model from the specified model configs
-        """
-
-        if not (model_config and isinstance(model_config, (ModelConfig, dict))):
-            msg = f"Need a valid model config to create a text/tagger model in AutoModel. " \
-                  f"Found model_config={model_config} of type({type(model_config)})"
-            raise ValueError(msg)
-
-        # get model type upon validation
-        model_config = cls._resolve_model_config(model_config)
-        model_type = cls._get_model_type(model_config)
-
-        # load metadata and return
-        if model_type == "text":
-            model_class = AutoTextModel.get_model_class(model_config)
-        elif model_type == "tagger":
-            model_class = AutoTaggerModel.get_model_class(model_config)
-
-        return model_class(model_config)
-
-    @classmethod
-    def from_path(cls, path: str) -> Union[None, Type[AbstractModel]]:
-        """
-        Loads a valid model from the specified path wherein the model was previously dumped.
-        Returns None when the specified path is not found or if the model loaded from the
-        specified path is a NoneType.
-        """
-
-        if not (path and isinstance(path, str)):
-            msg = f"Need a valid path to load a text/tagger model in AutoModel. " \
-                  f"Found path={path} of type({type(path)})"
-            raise ValueError(msg)
-
-        if not path.endswith(".pkl"):
-            msg = "Model Path must end with .pkl for AutoModel to be able to identify the model"
-            raise ValueError(msg)
-
-        try:
-            # if loading from path, determine the ABCModel type & return after doing xxxModel.load()
-            model_config = AbstractModel.load_model_config(path)
-
-            # get model type upon validation
-            model_config = cls._resolve_model_config(model_config)
-            model_type = cls._get_model_type(model_config)
-
-            # load metadata and return
-            if model_type == "text":
-                model_class = AutoTextModel.get_model_class(model_config)
-            elif model_type == "tagger":
-                model_class = AutoTaggerModel.get_model_class(model_config)
-
-            return model_class.load(path)
-
-        except FileNotFoundError:
-            # sometimes a model (and its config file) might not be dumped, eg. in role classifiers
-            # or even if dumped, can be of NoneType enclosed in a dictionary
-            return None
-
-    @staticmethod
-    def _resolve_model_config(model_config: Union[dict, ModelConfig]) -> ModelConfig:
-
-        # format configs
-        if isinstance(model_config, dict):
-            model_config = ModelConfig(**model_config)
-
-        # validate configs
-        if not isinstance(model_config, ModelConfig):
-            msg = f"Expected input config to be either a valid dictionary or an instance of " \
-                  f"ModelConfig class, but found of type {type(model_config)}"
-            raise ValueError(msg)
-
-        return model_config
-
-    @staticmethod
-    def _get_model_type(model_config: ModelConfig) -> str:
-
-        # identify the model_type
-        try:
-            model_type = model_config.model_type
-            assert model_type in AutoModel.ALLOWED_MODEL_TYPES
-        except (KeyError, AssertionError) as e:
-            msg = f"Invalid model configuration: Unknown model type {model_type}. " \
-                  f"Known types are: {AutoModel.ALLOWED_MODEL_TYPES}"
-            raise ValueError(msg) from e
-
-        return model_type
-
-    @staticmethod
-    def register_models() -> None:
-        for model_type in AutoModel.ALLOWED_MODEL_TYPES:
-            register_model(model_type, AutoModel)
-        register_model("auto", AutoModel)
-
-
-AutoModel.register_models()
diff --git a/mindmeld/models/containers.py b/mindmeld/models/containers.py
new file mode 100644
index 000000000..9574dd922
--- /dev/null
+++ b/mindmeld/models/containers.py
@@ -0,0 +1,463 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2015 Cisco Systems, Inc. and others.  All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+import os
+import zipfile
+from urllib.request import urlretrieve
+
+import numpy as np
+from tqdm import tqdm
+
+from ._util import _is_module_available, _get_module_or_attr as _getattr
+from ..core import Bunch
+from ..exceptions import EmbeddingDownloadError
+from ..path import (
+    EMBEDDINGS_FILE_PATH,
+    EMBEDDINGS_FOLDER_PATH,
+)
+from ..resource_loader import Hasher
+
+logger = logging.getLogger(__name__)
+
+
+class TqdmUpTo(tqdm):
+    """Provides `update_to(n)` which uses `tqdm.update(delta_n)`."""
+
+    def update_to(self, b=1, bsize=1, tsize=None):
+        """Reports update statistics on the download progress.
+
+        Args:
+            b (int): Number of blocks transferred so far [default: 1].
+            bsize (int): Size of each block (in tqdm units) [default: 1].
+            tsize (int): Total size (in tqdm units). If [default: None] remains unchanged.
+        """
+        if tsize is not None:
+            self.total = tsize
+        self.update(b * bsize - self.n)  # will also set self.n = b * bsize
+
+
+class GloVeEmbeddingsContainer:
+    """This class is responsible for the downloading, extraction and storing of
+    word embeddings based on the GloVe format.
+
+    To facilitate not loading the large glove embedding file to memory everytime a new container is
+    created, a class-level attribute with a hashmap is created.
+
+    TODO: refactor the call-signature similar to other containers by accepting
+    `pretrained_path_or_name` instead of token dimension and filepath. Also deprecate these two
+    arguments.
+    """
+    CONTAINER_LOOKUP = {}
+
+    GLOVE_DOWNLOAD_LINK = "http://nlp.stanford.edu/data/glove.6B.zip"
+    EMBEDDING_FILE_PATH_TEMPLATE = "glove.6B.{}d.txt"
+    ALLOWED_WORD_EMBEDDING_DIMENSIONS = [50, 100, 200, 300]
+
+    def __init__(self, token_dimension=300, token_pretrained_embedding_filepath=None):
+        # validations
+        if token_dimension not in GloVeEmbeddingsContainer.ALLOWED_WORD_EMBEDDING_DIMENSIONS:
+            logger.info(
+                "Token dimension %s not supported, "
+                "choose from these dimensions: %s. "
+                "Selected 300 by default",
+                token_dimension,
+                str(GloVeEmbeddingsContainer.ALLOWED_WORD_EMBEDDING_DIMENSIONS),
+            )
+            token_dimension = 300
+
+        self.token_dimension = token_dimension
+        self.token_pretrained_embedding_filepath = token_pretrained_embedding_filepath
+
+        # get model name hash
+        string_to_hash = json.dumps({"token_dimension": self.token_dimension})
+        self.model_id = Hasher(algorithm="sha1").hash(string=string_to_hash)
+
+        self._word_to_embedding = self._extract_embeddings()
+
+    def get_pretrained_word_to_embeddings_dict(self):
+        """Returns the word to embedding dict.
+
+        Returns:
+            (dict): word to embedding mapping.
+        """
+        return self._word_to_embedding
+
+    def _download_embeddings_and_return_zip_handle(self):
+
+        logger.info("Downloading embedding from %s", GloVeEmbeddingsContainer.GLOVE_DOWNLOAD_LINK)
+
+        # Make the folder that will contain the embeddings
+        if not os.path.exists(EMBEDDINGS_FOLDER_PATH):
+            os.makedirs(EMBEDDINGS_FOLDER_PATH)
+
+        with TqdmUpTo(
+            unit="B", unit_scale=True, miniters=1, desc=GloVeEmbeddingsContainer.GLOVE_DOWNLOAD_LINK
+        ) as t:
+
+            try:
+                urlretrieve(
+                    GloVeEmbeddingsContainer.GLOVE_DOWNLOAD_LINK, EMBEDDINGS_FILE_PATH,
+                    reporthook=t.update_to
+                )
+
+            except ConnectionError as e:
+                logger.error(
+                    "There was an issue downloading from this "
+                    "link %s with the following error: "
+                    "%s",
+                    GloVeEmbeddingsContainer.GLOVE_DOWNLOAD_LINK,
+                    e,
+                )
+                return
+
+            file_name = GloVeEmbeddingsContainer.EMBEDDING_FILE_PATH_TEMPLATE.format(
+                self.token_dimension)
+            zip_file_object = zipfile.ZipFile(EMBEDDINGS_FILE_PATH, "r")
+
+            if file_name not in zip_file_object.namelist():
+                logger.info(
+                    "Embedding file with %s dimensions not found in %s file path",
+                    self.token_dimension,
+                    file_name
+                )
+                return
+
+            return zip_file_object
+
+    @staticmethod
+    def _extract_and_map(glove_file):
+        word_to_embedding = {}
+        for line in glove_file:
+            values = line.split()
+            word = values[0]
+            if not isinstance(word, str):  # can be encoded as byte type
+                word = word.decode()
+            coefs = np.asarray(values[1:], dtype="float32")
+            word_to_embedding[word] = coefs
+        return word_to_embedding
+
+    def _extract_embeddings(self):
+        if self.model_id not in GloVeEmbeddingsContainer.CONTAINER_LOOKUP:
+
+            file_location = self.token_pretrained_embedding_filepath
+
+            if file_location and os.path.isfile(file_location):
+                msg = f"Extracting embeddings from provided file location {str(file_location)}."
+                logger.info(msg)
+                with open(file_location, "r") as embedding_file:
+                    word_to_embedding = self._extract_and_map(embedding_file)
+            else:
+                if file_location:
+                    logger.info("Provided file location %s does not exist.", str(file_location))
+                file_name = GloVeEmbeddingsContainer.EMBEDDING_FILE_PATH_TEMPLATE.format(
+                    self.token_dimension)
+                if os.path.isfile(EMBEDDINGS_FILE_PATH):
+                    msg = f"Extracting embeddings from default folder location " \
+                          f"{EMBEDDINGS_FILE_PATH}."
+                    logger.info(msg)
+                    try:
+                        zip_file_object = zipfile.ZipFile(EMBEDDINGS_FILE_PATH, "r")
+                        with zip_file_object.open(file_name) as embedding_file:
+                            word_to_embedding = self._extract_and_map(embedding_file)
+                    except zipfile.BadZipFile:
+                        logger.warning(
+                            "%s is corrupt. Deleting the zip file and attempting to"
+                            " download the embedding file again",
+                            EMBEDDINGS_FILE_PATH,
+                        )
+                        os.remove(EMBEDDINGS_FILE_PATH)
+                        self._extract_embeddings()
+                    except IOError as e:
+                        logger.error(
+                            "An error occurred when reading %s zip file. The file might"
+                            " be corrupt, so try deleting the file and running the program "
+                            "again",
+                            EMBEDDINGS_FILE_PATH,
+                        )
+                        raise IOError("Failed to load embeddings.") from e
+                else:
+                    logger.info("Default folder location %s does not exist.", EMBEDDINGS_FILE_PATH)
+                    zip_file_object = self._download_embeddings_and_return_zip_handle()
+                    if not zip_file_object:
+                        raise EmbeddingDownloadError("Failed to download embeddings.")
+                    with zip_file_object.open(file_name) as embedding_file:
+                        word_to_embedding = self._extract_and_map(embedding_file)
+
+            GloVeEmbeddingsContainer.CONTAINER_LOOKUP[self.model_id] = word_to_embedding
+
+        return GloVeEmbeddingsContainer.CONTAINER_LOOKUP[self.model_id]
+
+
+class SentenceTransformersContainer:
+    """This class is responsible for the downloading and extraction of sentence transformers models
+    based on the https://github.com/UKPLab/sentence-transformers format.
+
+    To facilitate not loading the large glove embedding file to memory everytime a new container is
+    created, a class-level attribute with a hashmap is created.
+    """
+    CONTAINER_LOOKUP = {}
+
+    def __init__(
+        self,
+        pretrained_name_or_abspath,
+        bert_output_type="mean",
+        quantize_model=False
+    ):
+        self.pretrained_name_or_abspath = pretrained_name_or_abspath
+        self.bert_output_type = bert_output_type
+        self.quantize_model = quantize_model
+
+        # get model name hash
+        string_to_hash = json.dumps({
+            "pretrained_name_or_abspath": self.pretrained_name_or_abspath,
+            "bert_output_type": self.bert_output_type,
+            "quantize_model": self.quantize_model,
+        }, sort_keys=True)
+        self.model_id = Hasher(algorithm="sha1").hash(string=string_to_hash)
+
+        self._model_bunch = None
+
+    def _extract_model(self):
+        """
+        Looks up the model in the class' lookup dict and returns the found value if available. If
+        not found, calls Huggingface APIs to download the model and then loads the model to the
+        lookup and returns the model.
+        """
+
+        model = SentenceTransformersContainer.CONTAINER_LOOKUP.get(self.model_id)
+
+        if not model:
+            info_msg = ""
+            for name in [
+                self.pretrained_name_or_abspath,
+                f"sentence-transformers/{self.pretrained_name_or_abspath}"
+            ]:
+                try:
+                    model = self._get_strans_encoder(
+                        name,
+                        output_type=self.bert_output_type,
+                        quantize=self.quantize_model
+                    )
+                    info_msg += f"Successfully initialized name/path `{name}` directly through " \
+                                f"huggingface-transformers. "
+                except OSError:
+                    info_msg += f"Could not initialize name/path `{name}` directly through " \
+                                f"huggingface-transformers. "
+                if model:
+                    break
+            logger.info(info_msg)
+
+            if not model:
+                msg = f"Could not resolve the name/path `{self.pretrained_name_or_abspath}`. " \
+                      f"Please check the model name and retry."
+                raise Exception(msg)
+
+            SentenceTransformersContainer.CONTAINER_LOOKUP[self.model_id] = model
+
+        return SentenceTransformersContainer.CONTAINER_LOOKUP[self.model_id]
+
+    @staticmethod
+    def _get_strans_encoder(
+        name_or_path,
+        output_type="mean",
+        quantize=False
+    ):
+        """
+        Retrieves a sentence-transformer model and returns it along with its transformer and
+        pooling components.
+
+        Args:
+            name_or_path: name or path to load a huggingface model
+            output_type: type of pooling required
+            quantize: if the model needs to be qunatized or not
+
+        Returns:
+            Bunch(sentence_transformers.Transformer,
+                  sentence_transformers.Pooling,
+                  sentence_transformers.SentenceTransformer)
+        """
+
+        if not _is_module_available("sentence_transformers"):
+            msg = "Must install extra [bert] by running " \
+                  "'pip install mindmeld[bert]'"
+            raise ImportError(msg)
+
+        strans_models = _getattr("sentence_transformers.models")
+        strans = _getattr("sentence_transformers", "SentenceTransformer")
+
+        transformer_model = strans_models.Transformer(
+            name_or_path,
+            model_args={"output_hidden_states": True}
+        )
+        pooling_model = strans_models.Pooling(
+            transformer_model.get_word_embedding_dimension(),
+            pooling_mode_cls_token=output_type == "cls",
+            pooling_mode_max_tokens=False,
+            pooling_mode_mean_tokens=output_type == "mean",
+            pooling_mode_mean_sqrt_len_tokens=False
+        )
+        sbert_model = strans(modules=[transformer_model, pooling_model])
+
+        if quantize:
+            if not _is_module_available("torch"):
+                raise ImportError("`torch` library required to quantize models") from None
+
+            torch_qint8 = _getattr("torch", "qint8")
+            torch_nn_linear = _getattr("torch.nn", "Linear")
+            torch_quantize_dynamic = _getattr("torch.quantization", "quantize_dynamic")
+
+            transformer_model = torch_quantize_dynamic(
+                transformer_model, {torch_nn_linear}, dtype=torch_qint8
+            ) if transformer_model else None
+            pooling_model = torch_quantize_dynamic(
+                pooling_model, {torch_nn_linear}, dtype=torch_qint8
+            ) if pooling_model else None
+            sbert_model = torch_quantize_dynamic(
+                sbert_model, {torch_nn_linear}, dtype=torch_qint8
+            ) if sbert_model else None
+
+        return Bunch(
+            transformer_model=transformer_model,
+            pooling_model=pooling_model,
+            sbert_model=sbert_model
+        )
+
+    def get_model_bunch(self):
+        # if already looked up, return it
+        if self._model_bunch:
+            return self._model_bunch
+        self._model_bunch = self._extract_model()
+        return self._model_bunch
+
+    def get_sbert_model(self):
+        return self.get_model_bunch().sbert_model
+
+    def get_pooling_model(self):
+        return self.get_model_bunch().pooling_model
+
+    def get_transformer_model(self):
+        return self.get_model_bunch().transformer_model
+
+
+class HuggingfaceTransformersContainer:
+    """This class is responsible for the downloading and extraction of transformers models such as
+     BERT, Multilingual-BERT, etc. based on the https://github.com/huggingface/transformers format.
+
+    To facilitate not loading the large glove embedding file to memory everytime a new container is
+    created, a class-level attribute with a hashmap is created.
+    """
+    CONTAINER_LOOKUP = {}
+
+    def __init__(
+        self,
+        pretrained_model_name_or_path,
+        quantize_model=False,
+        cache_lookup=True,
+        from_configs=False
+    ):
+
+        if not _is_module_available("transformers"):
+            msg = "Must install extra [transformers] by running " \
+                  "'pip install mindmeld[transformers]'"
+            raise ImportError(msg)
+
+        if from_configs:
+            if cache_lookup:
+                msg = "Cannot set both 'cache_lookup' and 'from_configs' to True at the same " \
+                      "time. Loading from Huggingface model configs returns a model without " \
+                      "pretrained weights' initialization."
+                raise ValueError(msg)
+            if quantize_model:
+                msg = "Huggingface model loaded from configs will be quantized instead of a " \
+                      "model loaded from pretrained weights"
+                logger.warning(msg)
+
+        self.pretrained_model_name_or_path = pretrained_model_name_or_path
+        self.quantize_model = quantize_model
+        self.cache_lookup = cache_lookup
+        self.from_configs = from_configs
+
+        # get model name hash
+        string_to_hash = json.dumps({
+            "pretrained_model_name_or_path": self.pretrained_model_name_or_path,
+            "quantize_model": self.quantize_model
+        }, sort_keys=True)
+        self.model_id = Hasher(algorithm="sha1").hash(string=string_to_hash)
+
+        self._model_bunch = None
+
+    def _extract_model(self):
+
+        if self.cache_lookup:
+            model = HuggingfaceTransformersContainer.CONTAINER_LOOKUP.get(self.model_id)
+        else:
+            model = None
+
+        if not model:
+
+            try:
+                if self.from_configs:
+                    config = self.get_transformer_model_config()
+                    transformer_model = _getattr("transformers", "AutoModel").from_config(config)
+                else:
+                    transformer_model = _getattr("transformers", "AutoModel").from_pretrained(
+                        self.pretrained_model_name_or_path)
+            except OSError as e:
+                msg = f"Could not resolve the name/path `{self.pretrained_model_name_or_path}`. " \
+                      f"Please check the model name/path and retry."
+                raise OSError(msg) from e
+
+            if self.quantize_model:
+                if not _is_module_available("torch"):
+                    raise ImportError("`torch` library required to quantize models") from None
+
+                torch_qint8 = _getattr("torch", "qint8")
+                torch_nn_linear = _getattr("torch.nn", "Linear")
+                torch_quantize_dynamic = _getattr("torch.quantization", "quantize_dynamic")
+
+                transformer_model = torch_quantize_dynamic(
+                    transformer_model, {torch_nn_linear}, dtype=torch_qint8)
+
+            model = Bunch(
+                config=self.get_transformer_model_config(),
+                tokenizer=self.get_transformer_model_tokenizer(),
+                transformer_model=transformer_model
+            )
+
+        # return the model without adding to lookup if the flag is set to False
+        if not self.cache_lookup:
+            return model
+
+        HuggingfaceTransformersContainer.CONTAINER_LOOKUP[self.model_id] = model
+        return HuggingfaceTransformersContainer.CONTAINER_LOOKUP[self.model_id]
+
+    def get_model_bunch(self):
+        # if already looked up/loaded, return it
+        if self._model_bunch:
+            return self._model_bunch
+        # either a bunch from lookup or newly loaded bunch if cache_lookup is False
+        self._model_bunch = self._extract_model()
+        return self._model_bunch
+
+    def get_transformer_model_config(self):
+        return _getattr("transformers", "AutoConfig").from_pretrained(
+            self.pretrained_model_name_or_path)
+
+    def get_transformer_model_tokenizer(self):
+        return _getattr("transformers", "AutoTokenizer").from_pretrained(
+            self.pretrained_model_name_or_path)
+
+    def get_transformer_model(self):
+        return self.get_model_bunch().transformer_model
diff --git a/mindmeld/models/helpers.py b/mindmeld/models/helpers.py
index 9edbd5850..c57c80f48 100644
--- a/mindmeld/models/helpers.py
+++ b/mindmeld/models/helpers.py
@@ -12,6 +12,7 @@
 # limitations under the License.
 
 """This module contains some helper functions for the models package"""
+import enum
 import json
 import logging
 import os
@@ -20,8 +21,9 @@
 
 import nltk
 from sklearn.metrics import make_scorer
-from ..text_preparation.text_preparation_pipeline import TextPreparationPipelineFactory
+
 from ..gazetteer import Gazetteer
+from ..text_preparation.text_preparation_pipeline import TextPreparationPipelineFactory
 
 logger = logging.getLogger(__name__)
 
@@ -66,6 +68,11 @@
 ]
 
 
+class ModelType(enum.Enum):
+    TEXT_MODEL = "text"
+    TAGGER_MODEL = "tagger"
+
+
 def create_model(config):
     """Creates a model instance using the provided configuration
 
@@ -79,7 +86,8 @@ def create_model(config):
         ValueError: When model configuration is invalid
     """
     try:
-        return MODEL_MAP["auto"].from_config(config)
+        # TODO: deprecate MODEL_MAP and use ModelFactory instead (be aware of cyclic imports)
+        return MODEL_MAP["auto"].create_model_from_config(config)
     except KeyError as e:
         msg = "Invalid model configuration: Unknown model type {!r}"
         raise ValueError(msg.format(config.model_type)) from e
@@ -98,7 +106,8 @@ def load_model(path):
     Raises:
         ValueError: When model configuration is invalid
     """
-    return MODEL_MAP["auto"].from_path(path)
+    # TODO: deprecate MODEL_MAP and use ModelFactory instead (be aware of cyclic imports)
+    return MODEL_MAP["auto"].create_model_from_path(path)
 
 
 def create_annotator(config):
@@ -191,6 +200,7 @@ def register_model(model_type, model_class):
         model_type (str): The model type as specified in model configs
         model_class (class): The model to register
     """
+    # TODO: deprecate MODEL_MAP var in in lieu of ModelFactory
     MODEL_MAP[model_type] = model_class
 
 
diff --git a/mindmeld/models/model.py b/mindmeld/models/model.py
index 231aee9f0..d47fe7169 100644
--- a/mindmeld/models/model.py
+++ b/mindmeld/models/model.py
@@ -12,6 +12,7 @@
 # limitations under the License.
 
 """This module contains base classes for models defined in the models subpackage."""
+
 import copy
 import json
 import logging
@@ -20,7 +21,7 @@
 import pickle
 from abc import ABC, abstractmethod
 from inspect import signature
-from typing import Union
+from typing import Union, Type, Dict, Any, Tuple, List, Pattern, Set
 
 from sklearn.externals import joblib
 from sklearn.model_selection import (
@@ -32,7 +33,10 @@
     StratifiedKFold,
     StratifiedShuffleSplit,
 )
+from sklearn.preprocessing import LabelEncoder as SKLabelEncoder
 
+from ._util import _is_module_available
+from .evaluation import EntityModelEvaluation, StandardModelEvaluation
 from .helpers import (
     CHAR_NGRAM_FREQ_RSC,
     ENABLE_STEMMING,
@@ -46,11 +50,23 @@
     get_label_encoder,
     ingest_dynamic_gazetteer,
 )
+from .nn_utils.helpers import EmbedderType
 from .._version import get_mm_version
-from ..text_preparation.text_preparation_pipeline import TextPreparationPipelineFactory
+from ..core import ProcessedQuery, QueryEntity
+from ..resource_loader import ResourceLoader, ProcessedQueryList as PQL
+from ..text_preparation.text_preparation_pipeline import (
+    TextPreparationPipelineFactory,
+    TextPreparationPipeline
+)
+
+# for backwards compatability for sklearn models serialized and dumped in previous version
+from .labels import LabelEncoder, EntityLabelEncoder  # pylint: disable=unused-import
 
 logger = logging.getLogger(__name__)
 
+Examples = Union[PQL.QueryIterator, PQL.ListIterator]
+Labels = Union[PQL.DomainIterator, PQL.IntentIterator, PQL.EntitiesIterator]
+
 
 class ModelConfig:
     """A value object representing a model configuration.
@@ -97,19 +113,18 @@ class ModelConfig:
 
     def __init__(
         self,
-        model_type=None,
-        example_type=None,
-        label_type=None,
-        features=None,
-        model_settings=None,
-        params=None,
-        param_selection=None,
-        train_label_set=None,
-        test_label_set=None,
+        model_type: str = None,
+        example_type: str = None,
+        label_type: str = None,
+        features: Dict = None,
+        model_settings: Dict = None,
+        params: Dict = None,
+        param_selection: Dict = None,
+        train_label_set: Pattern[str] = None,
+        test_label_set: Pattern[str] = None,
     ):
         for arg, val in {
             "model_type": model_type,
-            "example_type": example_type,
             "label_type": label_type,
         }.items():
             if val is None:
@@ -130,7 +145,7 @@ def __repr__(self):
         )
         return "{}({})".format(self.__class__.__name__, args_str)
 
-    def to_dict(self):
+    def to_dict(self) -> Dict:
         """Converts the model config object into a dict
 
         Returns:
@@ -141,7 +156,7 @@ def to_dict(self):
             result[attr] = getattr(self, attr)
         return result
 
-    def to_json(self):
+    def to_json(self) -> str:
         """Converts the model config object to JSON
 
         Returns:
@@ -149,7 +164,7 @@ def to_json(self):
         """
         return json.dumps(self.to_dict(), sort_keys=True)
 
-    def resolve_config(self, new_config):
+    def resolve_config(self, new_config: "ModelConfig"):
         """This method resolves any config incompatibility issues by
         loading the latest settings from the app config to the current config
 
@@ -163,7 +178,7 @@ def resolve_config(self, new_config):
         for setting in new_settings:
             setattr(self, setting, getattr(new_config, setting))
 
-    def get_ngram_lengths_and_thresholds(self, rname):
+    def get_ngram_lengths_and_thresholds(self, rname: str) -> Tuple:
         """
         Returns the n-gram lengths and thresholds to extract to optimize resource collection
 
@@ -204,7 +219,7 @@ def get_ngram_lengths_and_thresholds(self, rname):
 
         return lengths, thresholds
 
-    def required_resources(self):
+    def required_resources(self) -> Set:
         """Returns the resources this model requires
 
         Returns:
@@ -212,15 +227,23 @@ def required_resources(self):
         """
         # get list of resources required by feature extractors
         required_resources = set()
-        for name in self.features:
-            feature = get_feature_extractor(self.example_type, name)
-            required_resources.update(feature.__dict__.get("requirements", []))
+        if self.features:
+            for name in self.features:
+                feature = get_feature_extractor(self.example_type, name)
+                required_resources.update(feature.__dict__.get("requirements", []))
         return required_resources
 
 
 class AbstractModel(ABC):
     """
     A minimalistic abstract class upon which all models are based.
+
+    In order to maintain backwards compatability, the skeleton of this class is designed based on
+    all the access points of Classifier class and its sub-classes. In addition, it also introduces
+    the decoupled way of dumping/loading across different model types (meaning not all models are
+    dumped/loaded the same way). Furthermore, methods for validation are also introduced so as to
+    cater to the model specific config validations. Lastly, this skeleton also includes some
+    common properties that could be used across all model types.
     """
 
     def __init__(self, config: ModelConfig):
@@ -231,32 +254,40 @@ def __init__(self, config: ModelConfig):
         self._validate_model_configs()
 
     @abstractmethod
-    def initialize_resources(self, resource_loader, examples=None, labels=None):
+    def initialize_resources(
+        self, resource_loader: ResourceLoader, examples: Examples = None, labels: Labels = None
+    ):
         raise NotImplementedError
 
     @abstractmethod
-    def fit(self, examples, labels, params=None):
+    def fit(self, examples: Examples, labels: Labels, params: Dict = None):
         raise NotImplementedError
 
     @abstractmethod
-    def predict(self, examples, dynamic_resource=None):
+    def predict(
+        self, examples: Examples, dynamic_resource: Dict = None
+    ) -> Union[List[Any], List[List[Any]]]:
         raise NotImplementedError
 
     @abstractmethod
-    def predict_proba(self, examples):
+    def predict_proba(
+        self, examples: Examples, dynamic_resource: Dict = None
+    ) -> Union[List[Tuple[str, Dict[str, float]]], Tuple[Tuple[QueryEntity, float]]]:
         raise NotImplementedError
 
     @abstractmethod
-    def evaluate(self, examples, labels):
+    def evaluate(
+        self, examples: Examples, labels: Labels
+    ) -> Union[StandardModelEvaluation, EntityModelEvaluation]:
         raise NotImplementedError
 
     @classmethod
     @abstractmethod
-    def load(cls, path):
+    def load(cls, path: str) -> Type["AbstractModel"]:
         raise NotImplementedError
 
     @classmethod
-    def load_model_config(cls, path):
+    def load_model_config(cls, path: str) -> ModelConfig:
         """
         Dumps the model's configs. Raises a FileNotFoundError if no configs file is found.
         For backwards compatability wherein TextModel was serialized and dumped, the textModel file
@@ -300,7 +331,7 @@ def load_model_config(cls, path):
 
         return model_config
 
-    def dump(self, path) -> None:
+    def dump(self, path: str):
         """
         Dumps the model's configs and calls the child model's dump method
 
@@ -317,7 +348,7 @@ def dump(self, path) -> None:
         # (eg. serialized dump for sklearn-based models, .bin files for pytorch-based models, etc.)
         self._dump(path)
 
-    def _dump_model_config(self, path) -> None:
+    def _dump_model_config(self, path: str):
         """
         Dumps the model's configs
         """
@@ -326,7 +357,7 @@ def _dump_model_config(self, path) -> None:
         pickle.dump(self.config, open(model_configs_save_path, "wb"))
 
     @abstractmethod
-    def _dump(self, path) -> None:
+    def _dump(self, path: str):
         """
         Dumps the model and calls the underlying algo to dump its state.
 
@@ -335,8 +366,17 @@ def _dump(self, path) -> None:
         """
         pass
 
-    def view_extracted_features(self, example, dynamic_resource=None):
-        # Not implemeneted for deep neural models
+    @staticmethod
+    def _get_model_config_save_path(path: str) -> str:
+        head, ext = os.path.splitext(path)
+        model_config_save_path = head + ".config" + ext
+        os.makedirs(os.path.dirname(model_config_save_path), exist_ok=True)
+        return model_config_save_path
+
+    def view_extracted_features(
+        self, example: ProcessedQuery, dynamic_resource: Dict = None
+    ) -> List[Dict]:
+        # Not implemeneted unless overwritten by child class
         raise NotImplementedError
 
     def register_resources(self, **kwargs):  # pylint: disable=no-self-use
@@ -344,15 +384,22 @@ def register_resources(self, **kwargs):  # pylint: disable=no-self-use
         del kwargs
         pass
 
-    def get_resource(self, name):
+    def get_resource(self, name) -> Any:
         return self._resources.get(name)
 
-    @staticmethod
-    def _get_model_config_save_path(path):
-        head, ext = os.path.splitext(path)
-        model_config_save_path = head + ".config" + ext
-        os.makedirs(os.path.dirname(model_config_save_path), exist_ok=True)
-        return model_config_save_path
+    @property
+    def text_preparation_pipeline(self) -> TextPreparationPipeline:
+
+        text_preparation_pipeline = self._resources.get("text_preparation_pipeline")
+
+        if not text_preparation_pipeline:
+            logger.error(
+                "The text_preparation_pipeline resource has not been registered "
+                "to the model. Using default text_preparation_pipeline."
+            )
+            return TextPreparationPipelineFactory.create_default_text_preparation_pipeline()
+
+        return text_preparation_pipeline
 
     def _validate_model_configs(self):
         pass
@@ -367,6 +414,7 @@ class Model(AbstractModel):
 
     # model scoring type
     LIKELIHOOD_SCORING = "log_loss"
+    ALLOWED_CLASSIFIER_TYPES: List[str] = NotImplemented
 
     def __init__(self, config):
         super().__init__(config)
@@ -381,20 +429,6 @@ def _fit(self, examples, labels, params=None):
     def _get_model_constructor(self):
         raise NotImplementedError
 
-    @property
-    def text_preparation_pipeline(self):
-
-        text_preparation_pipeline = self._resources.get("text_preparation_pipeline")
-
-        if not text_preparation_pipeline:
-            logger.error(
-                "The text_preparation_pipeline resource has not been registered "
-                "to the model. Using default text_preparation_pipeline."
-            )
-            return TextPreparationPipelineFactory.create_default_text_preparation_pipeline()
-
-        return text_preparation_pipeline
-
     def _fit_cv(self, examples, labels, groups=None, selection_settings=None):
         """Called by the fit method when cross validation parameters are passed in. Runs cross
         validation and returns the best estimator and parameters.
@@ -712,7 +746,10 @@ def initialize_resources(self, resource_loader, examples=None, labels=None):
 
     def _validate_model_configs(self) -> Union[TypeError, ValueError]:
 
-        for arg, val in {"features": self.config.features, }.items():
+        for arg, val in {
+            "features": self.config.features,
+            "example_type": self.config.example_type
+        }.items():
             if val is None:
                 raise TypeError("__init__() missing required argument {!r}".format(arg))
 
@@ -725,28 +762,91 @@ def _validate_model_configs(self) -> Union[TypeError, ValueError]:
 
 
 class PytorchModel(AbstractModel):
+    ALLOWED_CLASSIFIER_TYPES: List[str] = NotImplemented  # to be implemented in child classes
+
+    def __init__(self, config):
+        if not _is_module_available('torch'):
+            raise ImportError("Install the extra 'torch' library by runnning "
+                              "'pip install mindmeld[torch]' to use pytorch based neural models")
 
-    def __init__(self, config):  # pylint: disable=useless-super-delegation
         super().__init__(config)
+        self._label_encoder = get_label_encoder(self.config)
+        self._class_encoder = SKLabelEncoder()
+        self._query_text_type = None
+        self._clf = None
 
     def initialize_resources(self, resource_loader, examples=None, labels=None):
-        raise NotImplementedError
+        del resource_loader, examples, labels
 
-    def fit(self, examples, labels, params=None):
-        raise NotImplementedError
+    @staticmethod
+    def _validate_training_data(examples: List[Any], labels: Union[List[int], List[List[int]]]):
+        if len(examples) != len(labels):
+            msg = f"Number of 'labels' ({len(labels)}) must be same as number of 'examples' " \
+                  f"({len(examples)})"
+            raise AssertionError(msg)
 
-    def predict(self, examples, dynamic_resource=None):
-        raise NotImplementedError
+    def _set_query_text_type(self, params: Dict = None, default: str = None):
+        """
+        Returns the query text type to use for obtaining training examples from Query objects.
 
-    def predict_proba(self, examples):
-        raise NotImplementedError
+        Args:
+            params (Dict, optional): The config params passed in to train the model
+            default (str, optional): The default text type to use in case no related configs found
+        """
 
-    def evaluate(self, examples, labels):
-        raise NotImplementedError
+        if params is None and self._query_text_type:
+            # this condition is satisfied during loading of models
+            return
 
-    @classmethod
-    def load(cls, path):
-        raise NotImplementedError
+        # While the key '_query_text_type' in config params allows for end-users to configure the
+        # choice of text_type to be used, it also needs the user to have knowledge about different
+        # text types in a Query object. Hence, this key is _underscored_ and is kept mainly for
+        # developer/benchmarking puposes.
+        query_text_type = params.get("_query_text_type", default) if params else default
+
+        # consider raw text for pretrained transformer models
+        if not query_text_type:
+            if params and EmbedderType(params.get("embedder_type")) == EmbedderType.BERT:
+                query_text_type = "text"
+
+        # if a NoneType value is found, use the processed_text by default
+        query_text_type = query_text_type or "processed_text"
 
-    def _dump(self, path):
+        # validation
+        allowed_text_types = ["text", "processed_text", "normalized_text"]
+        if query_text_type not in allowed_text_types:
+            msg = f"The params 'query_text_type' can only be among " \
+                  f"{allowed_text_types} but found value {query_text_type}."
+            logger.error(msg)
+            raise ValueError(msg)
+
+        self._query_text_type = query_text_type  # this var is dumped and loaded when loading models
+
+    def _get_texts_from_examples(self, examples: PQL.QueryIterator) -> List[str]:
+        """
+        Method that decides which text type- processed_text or raw_text -that needs to be used for
+        neural model training/inference.
+
+        Args:
+            examples (QueryIterator): A list of ProcessedQuery objects.
+
+        Returns:
+            texts (List[str]): A list of strings obtained from the query objects based on the
+                provided input configs
+        """
+        if not self._query_text_type:
+            msg = "The instance attribute '_query_text_type' must be set by calling " \
+                  "_set_query_text_type() method before calling the " \
+                  "_get_texts_from_examples() method."
+            raise ValueError(msg)
+        return [getattr(example, self._query_text_type) for example in examples]
+
+
+class AbstractModelFactory(ABC):
+    """
+    Abstract class for individual model factories like TextModelFactory and TaggerModelFactory
+    """
+
+    @abstractmethod
+    def get_model_cls(self, config: ModelConfig) -> Type[AbstractModel]:
         raise NotImplementedError
diff --git a/mindmeld/models/model_factory.py b/mindmeld/models/model_factory.py
new file mode 100644
index 000000000..e49849bf9
--- /dev/null
+++ b/mindmeld/models/model_factory.py
@@ -0,0 +1,192 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2015 Cisco Systems, Inc. and others.  All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains the ModelFactory class that can resolve model class names and create/load
+appropriate models
+"""
+
+import logging
+from typing import Union, Type
+
+from .helpers import register_model, ModelType
+from .model import ModelConfig, AbstractModel, AbstractModelFactory
+from .tagger_models import TaggerModelFactory
+from .text_models import TextModelFactory
+
+logger = logging.getLogger(__name__)
+
+
+class ModelFactory:
+    """Auto class that identifies appropriate text/tagger model from text_models.py/tagger_models.py
+    to load one based on the inputted configs or from the loaded configs file.
+
+    The .create_model_from_config() methods allows to load the appropriate model when a ModelConfig
+    is passed. The .create_model_from_path() method uses AbstractModel's load method to load a
+    dumped config, which is then used to load appropriate model and return it through a metadata
+    dictionary object.
+    """
+
+    def __new__(cls, config: Union[dict, ModelConfig]) -> Type[AbstractModel]:
+        # method for backwards compatibility in ./helpers/create_model()
+        return cls.create_model_from_config(config)
+
+    @classmethod
+    def create_model_from_config(
+        cls,
+        model_config: Union[dict, ModelConfig]
+    ) -> Type[AbstractModel]:
+        """
+        Instantiates and returns a valid model from the specified model configs
+
+        Args:
+            model_config (Union[dict, ModelConfig]): Model configs inputted either as dict or an
+                instance of ModelConfig
+
+        Returns:
+            model (Type[AbstractModel]): A text/tagger model instance
+
+        Raises:
+            ValueError: When the configs are invalid
+        """
+
+        is_valid_config = model_config and isinstance(model_config, (ModelConfig, dict))
+        if not is_valid_config:
+            msg = f"Need a valid model config to create a text/tagger model in ModelFactory. " \
+                  f"Found model_config={model_config} of type({type(model_config)})"
+            raise ValueError(msg)
+
+        model_config = cls._resolve_model_config(model_config)
+        model_type = cls._get_model_type(model_config)
+        model_class = cls._get_model_factory(model_type).get_model_cls(model_config)
+        return model_class(model_config)
+
+    @classmethod
+    def create_model_from_path(cls, path: str) -> Union[None, Type[AbstractModel]]:
+        """
+        Loads and returns a model from the specified path
+
+        Args:
+            path (str): A pickle file path from where a model can be loaded
+
+        Returns:
+            model (Union[None, Type[AbstractModel]]): Returns None when the specified path is not
+                found or if the model loaded from the specified path is a NoneType. If found a valid
+                config and a valid model, the model is load by calling .load() method and returned
+
+        Raises:
+            ValueError: When the path is invalid
+        """
+
+        if not (path and isinstance(path, str)):
+            msg = f"Need a valid path to load a text/tagger model in ModelFactory. " \
+                  f"Found path={path} of type({type(path)})"
+            raise ValueError(msg)
+
+        if not path.endswith(".pkl"):
+            msg = "Model Path must end with .pkl for ModelFactory to be able to identify the model"
+            raise ValueError(msg)
+
+        try:
+            # if loading from path, determine the ABCModel type & return after doing XxxModel.load()
+            model_config = AbstractModel.load_model_config(path)
+            model_config = cls._resolve_model_config(model_config)
+            model_type = cls._get_model_type(model_config)
+            model_class = cls._get_model_factory(model_type).get_model_cls(model_config)
+            return model_class.load(path)
+
+        except FileNotFoundError:
+            # sometimes a model (and its config file) might not be dumped, eg. in role classifiers
+            # or even if dumped, can be of NoneType enclosed in a dictionary
+            msg = f"No model file found while trying to load model from path: {path}. It might " \
+                  f"be the case that the classifier didn't need a model due to one or no classes."
+            logger.warning(msg)
+            return None
+
+    @staticmethod
+    def _resolve_model_config(model_config: Union[dict, ModelConfig]) -> ModelConfig:
+        """
+        Resolves and returns model configs.
+
+        Args:
+            model_config (Union[dict, ModelConfig]): If inputted as a dict, a new instance of
+                ModelConfig is created with that dict and returned.
+
+        Returns:
+            model_config (ModelConfig): An instance of ModelConfig
+
+        Raises:
+            ValueError: When the model config is of an invalid type
+        """
+
+        # format configs
+        if isinstance(model_config, dict):
+            model_config = ModelConfig(**model_config)
+
+        # validate configs
+        if not isinstance(model_config, ModelConfig):
+            msg = f"Expected input config to be either a valid dictionary or an instance of " \
+                  f"ModelConfig class, but found of type {type(model_config)}"
+            raise ValueError(msg)
+
+        return model_config
+
+    @staticmethod
+    def _get_model_type(model_config: ModelConfig) -> ModelType:
+        """
+        Returns model type from the model config and validates if it is a valid type or not.
+
+        Args:
+            model_config (ModelConfig): An instance of ModelConfig
+
+        Returns:
+            model_type (ModelType): The model type obtained from configs
+
+        Raises:
+            ValueError: When the model type is invalid
+        """
+
+        model_type = model_config.model_type
+
+        try:
+            return ModelType(model_type)
+        except ValueError as e:
+            msg = f"Invalid model configuration: Unknown model type {model_type}. " \
+                  f"Known types are: {[v.value for v in ModelType.__members__.values()]}"
+            raise ValueError(msg) from e
+
+    @staticmethod
+    def _get_model_factory(model_type: ModelType) -> Type[AbstractModelFactory]:
+        """
+        Returns a factory based on the provided model type
+
+        Args:
+            model_type (ModelType): An object of ModelType specifying the type of model to create
+
+        Returns:
+            model_factory (Type[AbstractModelFactory]): A model factory for specified model_type
+        """
+
+        return {
+            ModelType.TEXT_MODEL: TextModelFactory,
+            ModelType.TAGGER_MODEL: TaggerModelFactory,
+        }[model_type]
+
+    @staticmethod
+    def register_models() -> None:
+        for model_type in [v.value for v in ModelType.__members__.values()]:
+            register_model(model_type, ModelFactory)
+        register_model("auto", ModelFactory)
+
+
+ModelFactory.register_models()
diff --git a/mindmeld/models/nn_utils/__init__.py b/mindmeld/models/nn_utils/__init__.py
new file mode 100644
index 000000000..c3f550410
--- /dev/null
+++ b/mindmeld/models/nn_utils/__init__.py
@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2015 Cisco Systems, Inc. and others.  All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .helpers import (
+    TokenizerType,
+    EmbedderType,
+    SequenceClassificationType,
+    TokenClassificationType
+)
+from .sequence_classification import (
+    EmbedderForSequenceClassification,
+    CnnForSequenceClassification,
+    LstmForSequenceClassification,
+    BertForSequenceClassification,
+    get_sequence_classifier_cls
+)
+from .token_classification import (
+    EmbedderForTokenClassification,
+    LstmForTokenClassification,
+    CharCnnWithWordLstmForTokenClassification,
+    CharLstmWithWordLstmForTokenClassification,
+    BertForTokenClassification,
+    get_token_classifier_cls
+)
+
+__all__ = [
+    "EmbedderForSequenceClassification",
+    "CnnForSequenceClassification",
+    "LstmForSequenceClassification",
+    "BertForSequenceClassification",
+    "get_sequence_classifier_cls",
+    "EmbedderForTokenClassification",
+    "LstmForTokenClassification",
+    "CharCnnWithWordLstmForTokenClassification",
+    "CharLstmWithWordLstmForTokenClassification",
+    "BertForTokenClassification",
+    "get_token_classifier_cls",
+    "TokenizerType",
+    "EmbedderType",
+    "SequenceClassificationType",
+    "TokenClassificationType",
+]
diff --git a/mindmeld/models/nn_utils/classification.py b/mindmeld/models/nn_utils/classification.py
new file mode 100644
index 000000000..1ced2562b
--- /dev/null
+++ b/mindmeld/models/nn_utils/classification.py
@@ -0,0 +1,585 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2015 Cisco Systems, Inc. and others.  All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Base for custom modules that are developed on top of nn layers that can do
+sequence or token classification
+"""
+
+import json
+import logging
+import os
+import shutil
+import uuid
+from abc import abstractmethod
+from itertools import chain
+from typing import Dict, Union, List, Tuple
+
+import numpy as np
+from sklearn.metrics import accuracy_score, f1_score
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+
+from .helpers import (
+    BatchData,
+    get_disk_space_of_model,
+    get_num_weights_of_model,
+    get_default_params,
+    TokenizerType,
+    EmbedderType,
+    ValidationMetricType,
+    TRAIN_DEV_SPLIT_SEED,
+    LABEL_PAD_TOKEN_IDX,
+    DEFAULT_EMB_DIM,
+    DEFAULT_TOKENIZER
+)
+from .input_encoders import InputEncoderFactory
+from .._util import _get_module_or_attr
+from ..containers import GloVeEmbeddingsContainer
+from ...core import Bunch
+from ...path import USER_CONFIG_DIR
+
+try:
+    import torch
+    import torch.nn as nn
+
+    nn_module = _get_module_or_attr("torch.nn", "Module")
+except ImportError:
+    nn_module = object
+    pass
+
+logger = logging.getLogger(__name__)
+
+
+class BaseClassification(nn_module):
+    """
+    A base class for sequence & token classification using deep neural nets. Both the classification
+     submodules have a common fit() method defined in this base class, which also drives the
+     training of pytorch based deep nets. The net's computational graph is defined only when the
+     fit() method is called. This base class also holds few common utility methods and
+     further defines the skeleton of the children classes through abstract methods.
+    """
+
+    def __init__(self):
+        super().__init__()
+
+        self.name = self.__class__.__name__
+        self.params = Bunch()
+        self.params["name"] = self.name
+        self.encoder = None
+
+        self.ready = False  # True when .fit() is called or loaded from a checkpoint, else False
+        self.dirty = False  # True when the model weights aren't saved to disk yet, else False
+
+        self.out_dim = float('-inf')
+
+    def __repr__(self):
+        return f"<{self.name}> ready:{self.ready} dirty:{self.dirty}"
+
+    def get_default_params(self) -> Dict:
+        return get_default_params(self.__class__.__name__)
+
+    def log_and_return_model_info(self, verbose: bool = False) -> str:
+        """
+        Logs and returns the details of the underlying torch.nn model, such as occupying disk space
+        when dumped, number of parameters, the device on which the model is placed, etc.
+
+        Args:
+            verbose (bool): Determines the amount of information to be logged and returned.
+        """
+        msg = f"{self.name} " \
+              f"ready:{self.ready} dirty:{self.dirty} device:{self.params.device} " \
+              f"\n\tNumber of weights (trainable, all):{get_num_weights_of_model(self)} "
+        verbose_msg = msg + (
+            f"\n\tDisk Size (in MB): {get_disk_space_of_model(self):.4f} " if verbose else ""
+        )
+        logger.info(verbose_msg)
+        return msg
+
+    def to_device(self, batch_data: BatchData) -> BatchData:
+        """
+        Places pytorch tensors on the device configured through the params
+
+        Args:
+            batch_data (BatchData): A BatchData object consisting different tensor objects
+        """
+        for k, v in batch_data.items():
+            if v is not None and isinstance(v, torch.Tensor):
+                batch_data[k] = v.to(self.params.device)
+            elif isinstance(v, list):
+                batch_data[k] = [
+                    vv.to(self.params.device) if isinstance(vv, torch.Tensor) else vv
+                    for vv in v
+                ]
+            elif isinstance(v, dict):
+                batch_data[k] = self.to_device(batch_data[k])
+        return batch_data
+
+    # pylint: disable=too-many-locals
+    def fit(self, examples: List[str], labels: Union[List[int], List[List[int]]], **params):
+        """
+        Trains the underlying neural model on the inputted data and finally retains the best scored
+        model among all iterations.
+
+        Because of possibly large sized neural models, instead of retaining a copy of best set of
+        model weights on RAM, it is advisable to dump them in a temporary folder and upon completing
+        the training process, load the best checkpoint weights.
+
+        Args:
+            examples (List[str]): A list of text strings that will be used for model training and
+                validation
+            labels (Union[List[int], List[List[int]]]): A list of labels passed in as integers
+                corresponding to the examples. The encoded labels must have values between 0 and
+                n_classes-1 -- one label per example in case of sequence classification and a
+                sequence of labels per example in case of token classification
+        """
+
+        if self.ready:
+            msg = "The model is already fitted or is loaded from a file. Aborting re-fitting."
+            logger.error(msg)
+
+        # obtain and validate all parameters required to fit the model
+        params = {
+            **self.get_default_params(),
+            **params  # overwrite keys of default params that are passed-in
+        }
+        params = self._validate_and_update_params(**params)
+
+        # update params upon preparing encoder and embedder
+        params = self._prepare_input_encoder(examples, **params)
+        params = self._prepare_embedder(**params)
+
+        # update number of labels in params upon identifying the unique labels
+        try:
+            # labels for sequence classification are of type List[int] whereas for token
+            # classification they are List[List[int]]; this try-except tries to obtain the number
+            # of unique label strings for the purpose of classification
+            num_labels = len(set(labels))
+        except TypeError:
+            # error will be raised in case of token classification as `set(labels)` attempts to
+            # find a set of lists
+            num_labels = len(set(chain.from_iterable(labels)))
+        params.update({"num_labels": num_labels})
+
+        # update self.params which will be used throughout the following modeling code
+        self.params.update(params)
+
+        # init the graph and move model to device, inputs are moved to device on-the-go
+        self._init_graph()
+        self.to(self.params.device)
+
+        # dumping weights during training process into a temp folder instead of keeping in
+        # memory to reduce memory usage
+        temp_folder = os.path.join(USER_CONFIG_DIR, "tmp", "pytorch_models", str(uuid.uuid4()))
+        temp_weights_save_path = os.path.join(temp_folder, "pytorch_model.bin")
+        os.makedirs(temp_folder, exist_ok=True)
+
+        # split input data into train & dev splits, and get data loaders
+        train_examples, dev_examples, train_labels, dev_labels = train_test_split(
+            examples, labels, test_size=self.params.dev_split_ratio,
+            random_state=TRAIN_DEV_SPLIT_SEED
+        )
+
+        # create an optimizer and attach all model params to it
+        num_training_steps = int(
+            len(train_examples) / self.params.batch_size / self.params.gradient_accumulation_steps *
+            self.params.number_of_epochs
+        )
+        optimizer, scheduler = self._create_optimizer_and_scheduler(num_training_steps)
+
+        # set verbosity boolean
+        _verbose = (
+            logger.getEffectiveLevel() == logging.INFO or
+            logger.getEffectiveLevel() == logging.DEBUG
+        )
+        self.log_and_return_model_info(_verbose)
+
+        # training w/ validation
+        best_dev_score, best_dev_epoch = -np.inf, -1
+        msg = f"Beginning to train for {self.params.number_of_epochs} number of epochs"
+        logger.info(msg)
+        if self.params.number_of_epochs < 1:
+            raise ValueError("Param 'number_of_epochs' must be a positive integer greater than 0")
+        patience_counter = 0
+        for epoch in range(1, self.params.number_of_epochs + 1):
+            # patience before terminating due to no dev score improvements
+            if patience_counter >= self.params.patience:
+                msg = f"Set patience of {self.params.patience} epochs reached"
+                logger.info(msg)
+                break
+            # set modules to train phase, reset gradients, do forward-backward propagations
+            self.train()
+            optimizer.zero_grad()
+            train_loss, train_batches = 0.0, 0.0
+            t = tqdm(range(0, len(train_examples), self.params.batch_size), disable=not _verbose)
+            for start_idx in t:
+                batch_examples = train_examples[start_idx:start_idx + self.params.batch_size]
+                batch_labels = train_labels[start_idx:start_idx + self.params.batch_size]
+                batch_data = self.encoder.batch_encode(
+                    examples=batch_examples,
+                    padding_length=self.params.padding_length,
+                    add_terminals=self.params.add_terminals,
+                )
+                batch_data.update({
+                    "_labels": self._prepare_labels(  # `_` 'cause this key is for intermediate use
+                        batch_labels,
+                        # pad to the max length amongst encoded examples
+                        max([len(_split_lengths) for _split_lengths in batch_data["split_lengths"]])
+                    )
+                })
+                batch_data = self.forward(batch_data)
+                loss = batch_data["loss"]
+                train_loss += loss.cpu().detach().numpy()
+                train_batches += 1
+                # find gradients
+                loss = loss / self.params.gradient_accumulation_steps
+                loss.backward()
+                # optimizer and scheduler step
+                batch_id = start_idx / self.params.batch_size
+                if (
+                    start_idx + self.params.batch_size >= len(train_examples) or
+                    (batch_id + 1) % self.params.gradient_accumulation_steps == 0
+                ):
+                    # update weights when it is the last batch in the epoch or
+                    # when specified step is reached or
+                    if self.params.max_grad_norm:  # clip (accumulated) gradients if required
+                        nn.utils.clip_grad_norm_(self.parameters(), self.params.max_grad_norm)
+                    optimizer.step()
+                    scheduler.step()
+                    optimizer.zero_grad()
+                # log progress
+                progress_bar_msg = f"Epoch: {epoch} | Mean loss: {train_loss / train_batches:.4f}"
+                t.set_description(progress_bar_msg, refresh=True)
+            train_loss = train_loss / train_batches
+            # dev evaluation
+            predictions, targets = [], []
+            t = tqdm(range(0, len(dev_examples), self.params.batch_size), disable=not _verbose)
+            for start_idx in t:
+                batch_examples = dev_examples[start_idx:start_idx + self.params.batch_size]
+                batch_labels_targetted = dev_labels[start_idx:start_idx + self.params.batch_size]
+                batch_labels_predicted = self.predict(batch_examples)
+                # validation
+                if len(batch_labels_predicted) != len(batch_labels_targetted):
+                    msg = f"Number of predictions ({len(batch_labels_predicted)}) " \
+                          f"not equal to number of targets ({len(batch_labels_targetted)})"
+                    logger.error(msg)
+                    raise AssertionError(msg)
+                # flatten if required
+                try:
+                    batch_labels_predicted = sum(batch_labels_predicted, [])
+                    batch_labels_targetted = sum(batch_labels_targetted, [])
+                except TypeError:
+                    # raised in case of sequence classification; implies already flattened
+                    pass
+                # discard unwanted predictions using _label_padding_idx
+                batch_labels_predicted, batch_labels_targetted = zip(*[
+                    (x, y) for x, y in zip(batch_labels_predicted, batch_labels_targetted)
+                    if y != self.params._label_padding_idx
+                ])
+                predictions.extend(batch_labels_predicted)
+                targets.extend(batch_labels_targetted)
+                progress_bar_msg = f"Epoch: {epoch} | " \
+                                   f"Validation Metric: {self.params.validation_metric} "
+                t.set_description(progress_bar_msg, refresh=True)
+            # compute score
+            if ValidationMetricType(self.params.validation_metric) == ValidationMetricType.ACCURACY:
+                dev_score = accuracy_score(targets, predictions, normalize=True)
+            elif ValidationMetricType(self.params.validation_metric) == ValidationMetricType.F1:
+                dev_score = f1_score(targets, predictions, average='weighted')
+            else:
+                msg = f"Invalid 'validation_metric' ({self.params.validation_metric}) provided " \
+                      f"in params. Allowed values are only 'accuracy' and 'f1'"
+                raise ValueError(msg)
+            # save model weights in a temp folder; later move it to folder passed through dump()
+            if dev_score >= best_dev_score:
+                torch.save(self.state_dict(), temp_weights_save_path)
+                phrase = (
+                    f"improved from '{best_dev_score:.4f}' to" if dev_score > best_dev_score
+                    else "remained at"
+                )
+                msg = f"Model weights saved after epoch: {epoch} when dev score {phrase} " \
+                      f"'{dev_score:.4f}'\n"
+                logger.info(msg)
+                # update patience counter
+                if dev_score == best_dev_score:
+                    patience_counter += 1
+                else:
+                    patience_counter = 0
+                best_dev_score, best_dev_epoch = dev_score, epoch
+            else:
+                patience_counter += 1
+                msg = f"No weights saved after epoch: {epoch}. " \
+                      f"The dev score last improved after epoch: {best_dev_epoch}"
+                logger.info(msg)
+
+        # load back the best model dumped in temporary path and delete the temp folder
+        msg = f"Setting the model weights to checkpoint whose dev " \
+              f"{self.params.validation_metric} score is {best_dev_score:.4f}"
+        logger.info(msg)
+        # because we are loading to same device, no `map_location` specified
+        self.load_state_dict(torch.load(temp_weights_save_path))
+        shutil.rmtree(temp_folder)
+
+        self.ready = True
+        self.dirty = True
+
+    @staticmethod
+    def _validate_and_update_params(**params) -> Dict:
+        """Common validation and updation of the params dict before creating encoders and layers"""
+
+        # populate few required key-values (ensures the key-values are populated if not inputted)
+        params.update({
+            "add_terminals": params.get("add_terminals"),  # some encoders need this to be True
+            # (e.g. pretrained huggingface encoder) while some others don't; better not to set a
+            # boolean value as default to avoid raising errors unexpectedly
+            "padding_length": params.get("padding_length"),  # explicitly obtained for more
+            # transparent param dictionary
+            "tokenizer_type": params.get("tokenizer_type", DEFAULT_TOKENIZER),
+            "_label_padding_idx": LABEL_PAD_TOKEN_IDX,  # used to discard unwanted i.e. label
+            # padding indices in the batch predictions in the fit() method
+        })
+
+        # validate tokenizer_type param
+        allowed_tokenizer_types = {
+            EmbedderType.GLOVE: [
+                TokenizerType.WHITESPACE_TOKENIZER,
+                TokenizerType.WHITESPACE_AND_CHAR_DUAL_TOKENIZER,
+            ],
+            EmbedderType.BERT: [TokenizerType.HUGGINGFACE_PRETRAINED_TOKENIZER, ]
+        }
+        if params.get("embedder_type") and params.get("tokenizer_type"):
+            embedder_type = EmbedderType(params.get("embedder_type"))
+            tokenizer_type = TokenizerType(params.get("tokenizer_type"))
+            if embedder_type in allowed_tokenizer_types:
+                if tokenizer_type not in allowed_tokenizer_types[embedder_type]:
+                    msg = f"For the selected choice of embedder ({embedder_type.value}), only " \
+                          f"the following tokenizer_type are allowed: " \
+                          f"{[v.value for v in allowed_tokenizer_types[embedder_type]]}."
+                    raise ValueError(msg)
+
+        # validate validation metric
+        validation_metric = params.get("validation_metric")
+        try:
+            validation_metric = ValidationMetricType(validation_metric)
+        except ValueError as e:
+            msg = f"Expected validation_metric amongst " \
+                  f"{[v.value for v in ValidationMetricType.__members__.values()]} " \
+                  f"but found '{validation_metric}'."
+            raise ValueError(msg) from e
+
+        return params
+
+    def _prepare_input_encoder(self, examples: List[str], **params) -> Dict:
+        """Sets the input encoder and returns an updated param dict"""
+
+        # create and fit encoder
+        self.encoder = InputEncoderFactory.get_encoder_cls(params.get("tokenizer_type"))(**params)
+        self.encoder.prepare(examples=examples)
+        params.update({
+            "_num_tokens": len(self.encoder.get_vocab()),
+            "_padding_idx": self.encoder.get_pad_token_idx(),
+        })
+        return params
+
+    def _prepare_embedder(self, **params) -> Dict:
+        """Sets the embedder if required and returns an updated param dict"""
+
+        # check: cannot specify any conflicting params as required by child class
+        if self.encoder is None:
+            raise ValueError("An encoder must be first fitted before calling _prepare_embedder()")
+
+        # check: cannot specify any conflicting params as required by child class
+        embedder_type = params.get("embedder_type")
+        if EmbedderType(embedder_type) == EmbedderType.GLOVE:
+            # load glove embs
+            token_dimension = params.get("embedder_type", 300)
+            token_pretrained_embedding_filepath = params.get("token_pretrained_embedding_filepath")
+            glove_container = GloVeEmbeddingsContainer(
+                token_dimension=token_dimension,
+                token_pretrained_embedding_filepath=token_pretrained_embedding_filepath
+            )
+            token2emb = glove_container.get_pretrained_word_to_embeddings_dict()
+            glove_emb_dim = glove_container.token_dimension
+            # validate emb_dim
+            emb_dim = params.get("emb_dim", glove_emb_dim)
+            if emb_dim != glove_emb_dim:
+                msg = f"Provided 'emb_dim':{emb_dim} cannot be used with the provided " \
+                      f"'embedder_type':{embedder_type}. Consider not specifying any 'emb_dim' " \
+                      f"with this embedder."
+                raise ValueError(msg)
+            params.update({
+                "embedder_type": embedder_type,
+                "emb_dim": emb_dim,  # overwrite the default value
+                "_embedding_weights": {
+                    i: token2emb[t] for t, i in self.encoder.get_vocab().items() if t in token2emb
+                },
+            })
+        elif EmbedderType(embedder_type) == EmbedderType.BERT:
+            # the bert model is directly loaded in _init_core() itself
+            params.update({
+                "embedder_type": embedder_type,
+                "emb_dim": self.encoder.config.hidden_size,  # overwrite the default value
+                "pretrained_model_name_or_path": params.get("pretrained_model_name_or_path"),
+            })
+
+        if not params.get("emb_dim"):
+            msg = f"Need a valid 'emb_dim' to initialize embedding layers. To specify a " \
+                  f"particular dimension, either pass-in the 'emb_dim' param or provide a  valid " \
+                  f"'embedder_type' param. Continuing with a default value:{DEFAULT_EMB_DIM}."
+            logger.error(msg)
+            params.update({"emb_dim": DEFAULT_EMB_DIM})
+
+        return params
+
+    def _create_optimizer_and_scheduler(self, num_training_steps: int) -> Tuple:
+        """Sets an optimizer and scheduler for training torch.nn net"""
+        del num_training_steps
+
+        # load a torch optimizer
+        optimizer = getattr(torch.optim, self.params.optimizer)(
+            self.parameters(), lr=self.params.learning_rate
+        )
+        # load a constant lr scheduler
+        scheduler = getattr(torch.optim.lr_scheduler, "LambdaLR")(optimizer, lambda _: 1)
+        return optimizer, scheduler
+
+    def _get_dumpable_state_dict(self):
+        """
+        Returns a state dict of the Pytorch module that can be dumped. Overwriting definitions can
+        select a subset of full state dict to be dumped (e.g. like the BERT based ones)
+        """
+        return self.state_dict()
+
+    def dump(self, path: str):
+        """
+        Dumps underlying torch.nn model, encoder state and params
+
+        Args:
+            path (str): The path header for where the files are dumped.
+
+        The following states are dumped into different files:
+            - Pytorch model weights
+            - Encoder state
+            - Params (including params such as tokenizer_type and emb_dim that are used during
+                loading to create encoder and forward graph)
+        """
+        # resolve path and create associated folder if required
+        path = os.path.abspath(os.path.splitext(path)[0]) + ".pytorch_model"
+        os.makedirs(path, exist_ok=True)
+
+        # save weights
+        torch.save(self._get_dumpable_state_dict(), os.path.join(path, "model.bin"))
+
+        # save encoder's state
+        self.encoder.dump(path)
+
+        # save all params
+        with open(os.path.join(path, "params.json"), "w") as fp:
+            json.dump(dict(self.params), fp, indent=4)
+            fp.close()
+        msg = f"{self.name} model weights are dumped successfully"
+        logger.info(msg)
+
+        self.dirty = False
+
+    @classmethod
+    def load(cls, path: str):
+        """
+        Loads states from a dumped path
+
+        Args:
+            path (str): The path header wherein dumped files are present.
+        """
+        # resolve path
+        path = os.path.abspath(os.path.splitext(path)[0]) + ".pytorch_model"
+
+        # load all params
+        with open(os.path.join(path, "params.json"), "r") as fp:
+            all_params = json.load(fp)
+            fp.close()
+
+        # create new instance
+        module = cls()
+        if module.name != all_params["name"]:
+            msg = f"The name of the loaded model ({all_params['name']}) from the path '{path}' " \
+                  f"is different from the name of the module instantiated ({module.name})"
+            raise AssertionError(msg)
+
+        # load encoder's state
+        module.params.update(dict(all_params))
+        module.encoder = InputEncoderFactory.get_encoder_cls(
+            tokenizer_type=all_params["tokenizer_type"])(**all_params)
+        module.encoder.load(path)
+
+        # load weights
+        module._init_graph()
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        if device != module.params.device:
+            msg = f"Model was dumped when on the device:{module.params.device} " \
+                  f"but is not being loaded on device:{device}"
+            logger.warning(msg)
+            module.params.device = device
+        bin_path = os.path.join(path, "model.bin")
+        trained_state_dict = torch.load(bin_path, map_location=torch.device(device))
+        module_state_dict = module.state_dict()
+        keys_diff = module_state_dict.keys() - trained_state_dict.keys()
+        if keys_diff:
+            msg = f"While loading {module.__class__.__name__}, {len(keys_diff)} keys of the " \
+                  f"total {len(module_state_dict.keys())} of the torch module are not found in " \
+                  f"the file loaded from {bin_path} "
+            msg += "\n- This IS fine if loading a model for which only some parameters were " \
+                   "trained and others frozen. \n- This IS NOT fine if you expect all parameters " \
+                   "were trained."
+            logger.warning(msg)
+        module.load_state_dict(trained_state_dict, strict=False)
+        module.to(device)
+        msg = f"{module.name} model weights are loaded successfully on to the device:{device}"
+        logger.info(msg)
+
+        module.ready = True
+        module.dirty = False
+        return module
+
+    @abstractmethod
+    def _prepare_labels(self, labels: Union[List[int], List[List[int]]], max_length: int):
+        raise NotImplementedError
+
+    @abstractmethod
+    def _init_graph(self):
+        raise NotImplementedError
+
+    @abstractmethod
+    def forward(self, batch_data: BatchData) -> BatchData:
+        raise NotImplementedError
+
+    @abstractmethod
+    def predict(self, examples: List[str]) -> Union[List[int], List[List[int]]]:
+        """
+        Returns predicted class labels
+
+        Args:
+            examples (List[str]): The list of examples for which predictions are computed and
+                returned.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def predict_proba(self, examples: List[str]) -> Union[List[List[int]], List[List[List[int]]]]:
+        """
+        Returns predicted class probabilities
+
+        Args:
+            examples (List[str]): The list of examples for which class prediction probabilities
+                are computed and returned.
+        """
+        raise NotImplementedError
diff --git a/mindmeld/models/nn_utils/helpers.py b/mindmeld/models/nn_utils/helpers.py
new file mode 100644
index 000000000..a9b1eee51
--- /dev/null
+++ b/mindmeld/models/nn_utils/helpers.py
@@ -0,0 +1,322 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2015 Cisco Systems, Inc. and others.  All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Default params used by various sequence and token classification classes
+"""
+
+import enum
+import logging
+import os
+
+from .._util import _get_module_or_attr
+from ...core import Bunch
+
+try:
+    is_cuda_available = _get_module_or_attr("torch.cuda", "is_available")()
+except ImportError:
+    is_cuda_available = False
+    pass
+
+logger = logging.getLogger(__name__)
+
+
+class BatchData(Bunch):
+    """
+    A dictionary-like object that exposes its keys as attributes and holds various inputs as well
+    as outputs of neural models related to a batch of data, such as tensor encodings, lengths of
+    inputs etc.
+
+    Following is the description of the different keys that serve as inputs to neural models:
+        - seq_lengths: Number of tokens in each example before adding padding tokens. The number
+            includes terminal tokens too if they are added before padding. If using an encoder
+            that splits words in sub-words, seq_lengths still implies number of words (instead
+            of number of sub-words) along with any added terminal tokens; this number is
+            useful in case of token classifiers which require token-level (aka.
+            word-level) outputs as well as in sequence classifiers models such as LSTM.
+        - split_lengths: The length of each subgroup (i.e. group of sub-words) in each
+            example. Due to its definition, it obviously does not include any terminal
+            tokens in its counts. This can be seen as fine-grained information to
+            seq_lengths values for the encoders with sub-word tokenization. This is again
+            useful in cases of token classifiers to flexibly choose between representations
+            of first sub-word or mean/max pool of sub-words' representations in order to
+            obtain the word-level representations. For lookup table based encoders where
+            words are not broken into sub-words, `split_lengths` is simply a sequence of
+            ones whose sum indicates the number of words w/o terminal & padding tokens.
+        - seq_ids (in case non-pretrained models that require training an embedding layer):
+            The encoded ids useful for embedding lookup, including terminal special tokens
+            if asked for, and with padding.
+        - attention_masks (only in case of huggingface trainable encoders): Boolean flags
+            corresponding to each id in seq_ids, set to 0 if padding token else 1.
+        - hgf_encodings (only in huggingface pretrained encoders): A dict of outputs from a
+            Pretrained Language Model encoder from Huggingface (shortly dubbed as hgf).
+        - char_seq_ids (only in dual tokenizers): Similar to seq_ids but from a char
+            tokenizer in case of dual tokenization
+        - char_seq_lengths (only in dual tokenizers): Similar to seq_lengths but from a char
+            tokenizer in case of dual tokenization. Like seq_lengths, this also includes
+            terminal special tokens from char vocab in the length count whenever added.
+
+     Following is the description of the different keys that serve as inputs to neural models:
+
+    Following is the description of the different keys that are outputted by neural models:
+        - seq_embs: The embeddings produced before final classification (dense) layers by
+            sequence-classification classes (generally of shape [batch_size, emd_dim]).
+        - token_embs: The embeddings produced before final classification (dense) layers by
+            token-classification classes (generally of shape [batch_size, seq_length, emd_dim]).
+        - logits: Classification scores (before SoftMax).
+        - loss: Classification loss object.
+    """
+    pass
+
+
+class ValidationMetricType(enum.Enum):
+    ACCURACY = "accuracy"
+    F1 = "f1"
+
+
+class TokenizerType(enum.Enum):
+    WHITESPACE_TOKENIZER = "whitespace-tokenizer"
+    CHAR_TOKENIZER = "char-tokenizer"
+    WHITESPACE_AND_CHAR_DUAL_TOKENIZER = "whitespace_and_char-tokenizer"
+    BPE_TOKENIZER = "bpe-tokenizer"
+    WORDPIECE_TOKENIZER = "wordpiece-tokenizer"
+    HUGGINGFACE_PRETRAINED_TOKENIZER = "huggingface_pretrained-tokenizer"
+
+
+class EmbedderType(enum.Enum):
+    NONE = None
+    GLOVE = "glove"
+    BERT = "bert"
+
+
+class SequenceClassificationType(enum.Enum):
+    EMBEDDER = "embedder"
+    CNN = "cnn"
+    LSTM = "lstm"
+
+
+class TokenClassificationType(enum.Enum):
+    EMBEDDER = "embedder"
+    LSTM = "lstm-pytorch"
+    CNN_LSTM = "cnn-lstm"
+    LSTM_LSTM = "lstm-lstm"
+
+
+TRAIN_DEV_SPLIT_SEED = 6174
+LABEL_PAD_TOKEN_IDX = -1  # value set based on default label padding idx in pytorch
+
+DEFAULT_TRAINING_INFERENCE_PARAMS = {
+    "device": "cuda" if is_cuda_available else "cpu",
+    "number_of_epochs": 100,
+    "patience": 7,
+    "batch_size": 32,
+    "gradient_accumulation_steps": 1,
+    "max_grad_norm": None,
+    "optimizer": "Adam",
+    "learning_rate": 0.001,
+    "validation_metric": "accuracy",
+    "dev_split_ratio": 0.2
+}
+
+DEFAULT_VANILLA_BERT_MODEL_PARAMS = {
+    "batch_size": 16,
+    "gradient_accumulation_steps": 2,
+    "number_of_epochs": 20,
+    "patience": 5,
+    "embedder_output_pooling_type": "first",
+}
+
+DEFAULT_COMMON_TOKEN_CLASSIFICATION_PARAMS = {
+    "output_keep_prob": 0.7,
+    "use_crf_layer": True,
+    "patience": 10,  # observed in benchmarking that more patience is better when using crf
+    "token_spans_pooling_type":
+        "first",  # if words split in subgroups, tells which subword representation to consider
+    "validation_metric": "f1",
+}
+
+DEFAULT_EMB_DIM = 256
+DEFAULT_TOKENIZER = TokenizerType.WHITESPACE_TOKENIZER.value
+
+DEFAULT_FORWARD_PASS_PARAMS = {
+    "EmbedderForSequenceClassification": {
+        "embedder_type": EmbedderType.NONE.value,
+        "add_terminals": False,
+        "update_embeddings": True,
+        "embedder_output_keep_prob": 0.7,
+        "embedder_output_pooling_type": "mean",
+        "output_keep_prob": 1.0,  # set to 1.0 due to the shallowness of the architecture
+    },
+    "CnnForSequenceClassification": {
+        "embedder_type": EmbedderType.NONE.value,
+        "add_terminals": False,
+        "update_embeddings": True,
+        "embedder_output_keep_prob": 0.7,
+        "output_keep_prob": 0.7,
+        "window_sizes": [3, 4, 5],
+        "number_of_windows": [100, 100, 100],
+    },
+    "LstmForSequenceClassification": {
+        "embedder_type": EmbedderType.NONE.value,
+        "add_terminals": False,
+        "update_embeddings": True,
+        "embedder_output_keep_prob": 0.7,
+        "output_keep_prob": 0.7,
+        "lstm_hidden_dim": 128,
+        "lstm_num_layers": 2,
+        "lstm_keep_prob": 0.7,
+        "lstm_bidirectional": True,
+        "lstm_output_pooling_type": "last",
+    },
+    "BertForSequenceClassification": {
+        **DEFAULT_VANILLA_BERT_MODEL_PARAMS,
+        "pretrained_model_name_or_path": "bert-base-uncased",
+        "update_embeddings": True,
+        "embedder_output_keep_prob": 0.7,
+        "embedder_output_pooling_type": "first",
+        "output_keep_prob": 1.0,  # this dropout unnecessary upon using `embedder_output_keep_prob`
+        # other values for following keys will possibly throw errors
+        "embedder_type": "bert",
+        "add_terminals": True,
+        "tokenizer_type": TokenizerType.HUGGINGFACE_PRETRAINED_TOKENIZER.value,
+        # keys that are not mutually exclusive and are valid when some of the above keys are set
+        "save_frozen_bert_weights": False,  # the key is valid only when update_embeddings=True
+    },
+    "EmbedderForTokenClassification": {
+        **DEFAULT_COMMON_TOKEN_CLASSIFICATION_PARAMS,
+        "embedder_type": EmbedderType.NONE.value,
+        "update_embeddings": True,
+        "embedder_output_keep_prob": 0.7,
+        "output_keep_prob": 1.0,  # set to 1.0 due to the shallowness of the architecture,
+    },
+    "LstmForTokenClassification": {
+        **DEFAULT_COMMON_TOKEN_CLASSIFICATION_PARAMS,
+        "embedder_type": EmbedderType.NONE.value,
+        "update_embeddings": True,
+        "embedder_output_keep_prob": 0.7,
+        "lstm_hidden_dim": 128,
+        "lstm_num_layers": 2,
+        "lstm_keep_prob": 0.7,
+        "lstm_bidirectional": True,
+    },
+    "CharCnnWithWordLstmForTokenClassification": {
+        **DEFAULT_COMMON_TOKEN_CLASSIFICATION_PARAMS,
+        "embedder_type": EmbedderType.NONE.value,
+        "tokenizer_type": TokenizerType.WHITESPACE_AND_CHAR_DUAL_TOKENIZER.value,
+        "update_embeddings": True,
+        "embedder_output_keep_prob": 0.7,
+        "lstm_hidden_dim": 128,
+        "lstm_num_layers": 2,
+        "lstm_keep_prob": 0.7,
+        "lstm_bidirectional": True,
+        "char_emb_dim": 50,
+        "char_window_sizes": [3, 4, 5],
+        "char_number_of_windows": [100, 100, 100],
+        "char_cnn_output_keep_prob": 0.7,
+        "char_proj_dim": None,
+    },
+    "CharLstmWithWordLstmForTokenClassification": {
+        **DEFAULT_COMMON_TOKEN_CLASSIFICATION_PARAMS,
+        "tokenizer_type": TokenizerType.WHITESPACE_AND_CHAR_DUAL_TOKENIZER.value,
+        "update_embeddings": True,
+        "embedder_output_keep_prob": 0.7,
+        "lstm_hidden_dim": 128,
+        "lstm_num_layers": 2,
+        "lstm_keep_prob": 0.7,
+        "lstm_bidirectional": True,
+        "char_emb_dim": 50,
+        "char_lstm_hidden_dim": 128,
+        "char_lstm_num_layers": 2,
+        "char_lstm_keep_prob": 0.7,
+        "char_lstm_bidirectional": True,
+        "char_lstm_output_pooling_type": "last",
+        "char_proj_dim": None,
+    },
+    "BertForTokenClassification": {
+        **DEFAULT_VANILLA_BERT_MODEL_PARAMS,
+        **DEFAULT_COMMON_TOKEN_CLASSIFICATION_PARAMS,
+        "pretrained_model_name_or_path": "bert-base-uncased",
+        "update_embeddings": True,
+        "embedder_output_keep_prob": 0.7,
+        "output_keep_prob": 1.0,  # this dropout unnecessary upon using `embedder_output_keep_prob`
+        "use_crf_layer": False,  # Following BERT paper's best results,
+        # other values for following keys will possibly throw errors
+        "embedder_type": "bert",
+        "add_terminals": True,
+        "tokenizer_type": TokenizerType.HUGGINGFACE_PRETRAINED_TOKENIZER.value,
+        # keys that are not mutually exclusive and are valid when some of the above keys are set
+        "save_frozen_bert_weights": False,  # the key is valid only when update_embeddings=True
+    }
+}
+
+
+def get_default_params(class_name: str):
+    """
+    Returns all the default params based on the inputted class name
+
+    Args:
+        class_name (str): A (child) class name from sequence_classification.py or
+            token_classification.py
+    """
+    try:
+        return {
+            **DEFAULT_TRAINING_INFERENCE_PARAMS,
+            **DEFAULT_FORWARD_PASS_PARAMS[class_name]
+        }
+    except KeyError as e:
+        msg = f"Cannot find module name {class_name} when looking for default params."
+        logger.error(msg)
+        raise KeyError(msg) from e
+
+
+def get_disk_space_of_model(pytorch_module):
+    """
+    Returns the disk space of a pytorch module in MB units. This includes all weights
+    (trainable and non-trainable) of the module.
+
+    Args:
+        pytorch_module: a pytorch neural network module derived from torch.nn.Module
+
+    Returns:
+        size (float): The size of model when dumped
+    """
+    filename = "temp.bin"
+    _get_module_or_attr("torch").save(pytorch_module.state_dict(), filename)
+    size = os.path.getsize(filename) / 1e6
+    os.remove(filename)
+    msg = f"Pytorch module will be dumped temporarily at {filename} in order to " \
+          f"calculate its disk size."
+    logger.debug(msg)
+    return size
+
+
+def get_num_weights_of_model(pytorch_module):
+    """
+    Returns the number of trainable and the total parameters in a pytorch module. Returning both
+    helps to do a sanity check if any layers which are meant to be frozen are being trained or not.
+
+    Args:
+        pytorch_module: a pytorch neural network module derived from torch.nn.Module
+
+    Returns:
+        number_of_params (tuple): A tuple of number of params that are trainable and total number
+            of params of the pytorch module
+    """
+    n_total = 0
+    n_requires_grad = 0
+    for param in list(pytorch_module.parameters()):
+        t = param.numel()
+        n_total += t
+        if param.requires_grad:
+            n_requires_grad += t
+    return n_requires_grad, n_total
diff --git a/mindmeld/models/nn_utils/input_encoders.py b/mindmeld/models/nn_utils/input_encoders.py
new file mode 100644
index 000000000..9a16cf0a8
--- /dev/null
+++ b/mindmeld/models/nn_utils/input_encoders.py
@@ -0,0 +1,903 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2015 Cisco Systems, Inc. and others.  All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module consists of encoders that serve as input to pytorch modules
+"""
+import json
+import logging
+import os
+from abc import abstractmethod, ABC
+from itertools import chain
+from typing import Dict, List, Union, Any, Tuple
+
+from .helpers import BatchData, TokenizerType
+from .._util import _get_module_or_attr
+from ..containers import HuggingfaceTransformersContainer
+
+try:
+    import torch
+except ImportError:
+    pass
+
+try:
+    from tokenizers import normalizers
+    from tokenizers.trainers import Trainer
+    from tokenizers.normalizers import Lowercase, NFD, StripAccents
+    from tokenizers.pre_tokenizers import Whitespace
+    from tokenizers.models import BPE, WordPiece
+    from tokenizers.trainers import BpeTrainer, WordPieceTrainer
+    from tokenizers.processors import TemplateProcessing
+
+    NO_TOKENIZERS_MODULE = False
+except ImportError:
+    NO_TOKENIZERS_MODULE = True
+    pass
+
+logger = logging.getLogger(__name__)
+
+
+class AbstractEncoder(ABC):
+    """
+    Defines a stateful tokenizer. Unlike the tokenizer in the text_preperation_pipeline, tokenizers
+    derived from this abstract class have a state such a vocabulary or a trained/pretrained model
+    that is used for encoding an input textual string into sequence of ids or a sequence of
+    embeddings. These outputs are used by the initial layers of neural nets.
+    """
+
+    def __init__(self, **_kwargs):
+        ununsed_kwargs = ','.join([f'{k}:{v}' for k, v in _kwargs.items()])
+        msg = f"The following keyword arguments are not used while initializing " \
+              f"{self.__class__.__name__}: {ununsed_kwargs}"
+        logger.debug(msg)
+
+    @abstractmethod
+    def prepare(self, examples: List[str]):
+        """
+        Method that fits the tokenizer and creates a state that can be dumped or used for encoding
+
+        Args:
+            examples: List of text strings that will be used for creating the state of the tokenizer
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+    @abstractmethod
+    def dump(self, path: str):
+        """
+        Method that dumps the state (if any) of the tokenizer
+
+        Args:
+            path: The folder where the state has to be dumped
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+    @abstractmethod
+    def load(self, path: str):
+        """
+        Method that dumps the state (if any) of the tokenizer
+
+        Args:
+            path: The folder where the dumped state can be found. Not all tokenizers dump with same
+                file names, hence we use a folder name rather than filename.
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+    @abstractmethod
+    def _tokenize(self, text: str) -> List[str]:
+        """
+        Method that converts a peice of text into a sequence of strings
+
+        Args:
+            text (str): Input text.
+
+        Returns:
+            tokens (List[str]): List of tokens.
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+    @abstractmethod
+    def batch_encode(
+        self, examples: List[str], padding_length: int = None, add_terminals: bool = False, **kwargs
+    ) -> BatchData:
+        """
+        Method that encodes a list of texts into a list of sequence of ids
+
+        Args:
+            examples: List of text strings that will be encoded as a batch
+            padding_length: The maximum length of each encoded input. Sequences less than this
+                length are padded to padding_length, longer sequences are trimmed. If not specified,
+                the max length of examples upon tokenization is used as padding_length.
+            add_terminals: A boolean flag that determines if terminal special tokens are to be added
+                to the tokenized examples or not.
+
+        Returns:
+            BatchData: A dictionary-like object for the supplied batch of data, consisting of
+                various tensor inputs to the neural computation graph as well as any other inputs
+                required during the forward computation.
+
+        Special note on `add_terminals` when using for sequence classification:
+            This flag can be True or False in general. Setting it to False will lead to errors in
+            case of Huggingface tokenizers as they are generally built to include terminals along
+            with pad tokens. Hence, the default value for `add_terminals` is False in case of
+            encoders built on top of AbstractVocabLookupEncoder and True for Hugginface ones. This
+            value can be True or False for encoders based on AbstractVocabLookupEncoder for sequence
+            classification.
+        Special note on `add_terminals` when using for token classification:
+            When using a CRF layer, `add_terminals` must be set to False and hence Huggingface
+            encoders cannot be used in such scenarios. If not using a CRF layer, one can use any
+            encoders by setting it to True but care must be taken to modify labels accordingly.
+            Hence, it is advisable to use `split_lengths` for padding labels in case of token
+            classification instead of `seq_lengths`
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+    @abstractmethod
+    def get_vocab(self) -> Dict:
+        """Returns a dictionary of vocab tokens as keys and their ids as values"""
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def get_pad_token_idx(self) -> Union[None, int]:
+        """
+        If there exists a padding token's index in the vocab, it is returned; useful while
+        initializing an embedding layer. Else returns a None.
+        """
+        if not hasattr(self, "pad_token_idx"):
+            return None
+        return getattr(self, "pad_token_idx")
+
+    @property
+    def number_of_terminal_tokens(self) -> int:
+        """
+        Returns the number of terminal tokens used by the encoder during batch encoding when
+        add_terminals is set to True.
+        """
+        return 2
+
+
+def _trim_a_list_of_sub_token_groups(
+    x: List[List[Any]],
+    max_len: int,
+    y: List[Any] = None
+) -> Union[Tuple[List[Any], List], List[Any]]:
+    """
+    Given a list of sub-tokens sequences (aka. groups) upon a tokenization step, this method
+    identifies the first N groups that can be consumed within the allowed max length max_len.
+
+    Args:
+        x: List of groups of sub-words, obtained upon whitespace pre-tokenization and word-level
+            tokenization using a huggingface tokenizer
+        max_len: The maximum length of ravelled output expected. If given a value greater than the
+            number of all sub-words inputted, it is clipped to number of all sub-words.
+        y: Labels accompanying each group in x
+    """
+    max_len = min(max_len, sum([len(_x) for _x in x]))
+    curr_len = 0
+    if y:
+        new_x, new_y = [], []
+        # iter through each sub-tokens group w/ respective
+        # group's label (eg. group: ["m", "##ug"])
+        for _x, _y in zip(x, y):
+            if curr_len + len(_x) > max_len:
+                return new_x, new_y
+            elif curr_len + len(_x) <= max_len:
+                new_x.append(_x)
+                new_y.append(_y)
+                curr_len += len(_x)
+        return new_x, new_y
+    else:
+        new_x = []
+        # iter through each sub-tokens group w/o respective
+        # group's label (eg. group: ["m", "##ug"])
+        for _x in x:
+            if curr_len + len(_x) > max_len:
+                return new_x
+            elif curr_len + len(_x) <= max_len:
+                new_x.append(_x)
+                curr_len += len(_x)
+        return new_x
+
+
+class AbstractVocabLookupEncoder(AbstractEncoder):
+    """
+    Abstract class wrapped around AbstractEncoder that has a vocabulary lookup as the state.
+    """
+
+    SPECIAL_TOKENS_DICT = {
+        "pad_token": "<PAD>",
+        "unk_token": "<UNK>",
+        "start_token": "<START>",
+        "end_token": "<END>",
+    }
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.token2id = {}
+
+    @property
+    def id2token(self):
+        return {i: t for t, i in self.token2id.items()}
+
+    def prepare(self, examples: List[str]):
+        examples = [ex.strip() for ex in examples]
+        all_tokens = dict.fromkeys(chain.from_iterable([self._tokenize(text) for text in examples]))
+        self.token2id = {t: i for i, t in enumerate(all_tokens)}
+
+        for name, token in self.__class__.SPECIAL_TOKENS_DICT.items():
+            self.token2id.update({token: len(self.token2id)})
+            setattr(self, f"{name}", token)
+            setattr(self, f"{name}_idx", self.token2id[token])
+
+    def dump(self, path: str):
+        filename = self._get_filename(path)
+        os.makedirs(os.path.dirname(filename), exist_ok=True)
+        with open(filename, "w") as opfile:
+            for token in self.token2id:
+                opfile.write(f"{token}\n")
+        with open(os.path.join(path, "special_vocab.json"), "w") as fp:
+            json.dump(self.__class__.SPECIAL_TOKENS_DICT, fp, indent=4)
+        msg = f"The state of {self.__class__.__name__} is successfully dumped at '{filename}'"
+        logger.info(msg)
+
+    def load(self, path: str):
+        filename = self._get_filename(path)
+        with open(filename, "r") as opfile:
+            tokens = [line.strip() for line in opfile]
+            self.token2id = {t: i for i, t in enumerate(tokens)}
+        with open(os.path.join(path, "special_vocab.json"), "r") as fp:
+            special_tokens_dict = json.load(fp)
+            for name, token in special_tokens_dict.items():
+                setattr(self, f"{name}", token)
+                setattr(self, f"{name}_idx", self.token2id[token])
+
+        msg = f"The state of {self.__class__.__name__} is successfully loaded from '{filename}'"
+        logger.info(msg)
+
+    def _get_filename(self, path: str):
+        if not os.path.isdir(path):
+            msg = f"The dump method of {self.__class__.__name__} only accepts diretory as the " \
+                  f"path argument."
+            logger.error(msg)
+            raise ValueError(msg)
+        filename = os.path.join(path, "vocab.txt")
+        return filename
+
+    @abstractmethod
+    def _tokenize(self, text: str) -> List[str]:
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def batch_encode(
+        self, examples: List[str], padding_length: int = None, add_terminals: bool = False,
+        _return_tokenized_examples: bool = False, **kwargs
+    ) -> BatchData:
+
+        n_terminals = self.number_of_terminal_tokens if add_terminals else 0
+
+        # tokenize each word of each input separately
+        # get maximum length of each example, accounting for terminal tokens- cls, sep
+        # If padding_length is None, padding has to be done to the max length of the input batch
+        tokenized_examples = [
+            [self._tokenize(word) for word in example.split(" ")] for example in examples
+        ]
+
+        max_curr_len = max([len(sum(t_ex, [])) for t_ex in tokenized_examples]) + n_terminals
+        padding_length_including_terminals = min(max_curr_len, padding_length) \
+            if padding_length else max_curr_len
+        padding_length_excluding_terminals = padding_length_including_terminals - n_terminals
+
+        _trimmed_examples = [
+            _trim_a_list_of_sub_token_groups(tokenized_example, padding_length_excluding_terminals)
+            for tokenized_example in tokenized_examples
+        ]  # List[List[List[str]]], innermost List[str] is a list of sub-words for a given word
+
+        split_lengths = [[len(x) for x in ex] for ex in _trimmed_examples]
+
+        # convert (sub) words into their respective token ids
+        seq_ids = [
+            self._encode_text(sum(example, []), padding_length_including_terminals, add_terminals)
+            for example in tokenized_examples
+        ]
+
+        return BatchData(**{
+            # number of groups per example
+            "seq_lengths": torch.as_tensor(  # Tensor1d[int]
+                [len(_split_lengths) + n_terminals for _split_lengths in split_lengths],
+                dtype=torch.long
+            ),
+            # len of each subgroup; for each example, sum of its split_lengths will be equal to
+            # the sequence length minus terminals.
+            "split_lengths": [
+                torch.as_tensor(_split_lengths, dtype=torch.long)
+                for _split_lengths in split_lengths
+            ],  # List[Tensor1d[int]],
+            "seq_ids": torch.as_tensor(seq_ids, dtype=torch.long),
+            **(
+                {"_examples": [sum(list_of_t, []) for list_of_t in _trimmed_examples]}
+                if _return_tokenized_examples else {}
+            ),
+        })
+
+    def _encode_text(self, list_of_tokens: List[str], padding_length: int, add_terminals: bool):
+        """
+        Encodes a list of tokens in to a list of ids based on vocab and special token ids.
+
+        Args:
+            list_of_tokens (List[str]): List of words or sub-words that are to be encoded
+            padding_length (int): Maximum length of the encoded sequence; sequences shorter than
+                this length are padded with a pad index while longer sequences are trimmed.
+                Upon encoding, the length of outputted list of ids will be this length.
+            add_terminals (bool): Whether terminal start and end tokens are to be added or not to
+                the encoded sequence
+
+        Returns:
+            list_of_ids (List[int]): Sequence of ids corresponding to the input tokens
+        """
+        list_of_tokens_with_terminals = (
+            [getattr(self, "start_token")] +
+            list_of_tokens +
+            [getattr(self, "end_token")]
+        ) if add_terminals else list_of_tokens
+        list_of_tokens_with_terminals_and_padding = (
+            list_of_tokens_with_terminals +
+            [getattr(self, "pad_token")] * (padding_length - len(list_of_tokens_with_terminals))
+        )
+        list_of_ids = [
+            self.token2id.get(token, getattr(self, "unk_token_idx"))
+            for token in list_of_tokens_with_terminals_and_padding
+        ]
+        return list_of_ids
+
+    def get_vocab(self) -> Dict:
+        return self.token2id
+
+
+class WhitespaceEncoder(AbstractVocabLookupEncoder):
+    """
+    Encoder that tokenizes at whitespace. Not useful for languages such as Chinese.
+    """
+
+    def _tokenize(self, text: str) -> List[str]:
+        return text.strip("\n").split(" ")
+
+
+class CharEncoder(AbstractVocabLookupEncoder):
+    """
+    A simple tokenizer that tokenizes at character level
+    """
+
+    def _tokenize(self, text: str) -> List[str]:
+        return list(text.strip("\n"))
+
+
+class WhitespaceAndCharDualEncoder(WhitespaceEncoder):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.char_token2id = {}
+
+    SPECIAL_CHAR_TOKENS_DICT = {
+        "char_pad_token": "<CHAR_PAD>",
+        "char_unk_token": "<CHAR_UNK>",
+        "char_start_token": "<CHAR_START>",
+        "char_end_token": "<CHAR_END>",
+    }
+
+    @property
+    def char_id2token(self):
+        return {i: t for t, i in self.char_token2id.items()}
+
+    @staticmethod
+    def char_tokenize(text: str) -> List[str]:
+        return list(text.strip("\n"))
+
+    def prepare(self, examples: List[str]):
+        super().prepare(examples)
+
+        examples = [ex.strip() for ex in examples]
+        all_tokens = dict.fromkeys(
+            chain.from_iterable([self.char_tokenize(text) for text in examples])
+        )
+        self.char_token2id = {t: i for i, t in enumerate(all_tokens)}
+
+        for name, token in self.__class__.SPECIAL_CHAR_TOKENS_DICT.items():
+            self.char_token2id.update({token: len(self.char_token2id)})
+            setattr(self, f"{name}", token)
+            setattr(self, f"{name}_idx", self.char_token2id[token])
+
+    def dump(self, path: str):
+        super().dump(path)
+
+        filename = self._get_char_filename(path)
+        os.makedirs(os.path.dirname(filename), exist_ok=True)
+        with open(filename, "w") as opfile:
+            for token in self.char_token2id:
+                opfile.write(f"{token}\n")
+        with open(os.path.join(path, "special_char_vocab.json"), "w") as fp:
+            json.dump(self.__class__.SPECIAL_CHAR_TOKENS_DICT, fp, indent=4)
+        msg = f"The state of {self.__class__.__name__} is successfully dumped at '{filename}'"
+        logger.info(msg)
+
+    def load(self, path: str):
+        super().load(path)
+
+        filename = self._get_char_filename(path)
+        with open(filename, "r") as opfile:
+            tokens = [line.strip() for line in opfile]
+            self.char_token2id = {t: i for i, t in enumerate(tokens)}
+        with open(os.path.join(path, "special_char_vocab.json"), "r") as fp:
+            special_char_tokens_dict = json.load(fp)
+            for name, token in special_char_tokens_dict.items():
+                setattr(self, f"{name}", token)
+                setattr(self, f"{name}_idx", self.char_token2id[token])
+
+        msg = f"The state of {self.__class__.__name__} is successfully loaded from '{filename}'"
+        logger.info(msg)
+
+    def _get_char_filename(self, path: str):
+        if not os.path.isdir(path):
+            msg = f"The dump method of {self.__class__.__name__} only accepts diretory as the " \
+                  f"path argument."
+            logger.error(msg)
+            raise ValueError(msg)
+        filename = os.path.join(path, "char_vocab.txt")
+        return filename
+
+    @property
+    def number_of_char_terminal_tokens(self) -> int:
+        """
+        Returns the number of char terminal tokens used by the encoder during batch encoding when
+        add_terminals is set to True
+        """
+        return 2
+
+    @property
+    def number_of_terminal_tokens(self) -> int:
+        """
+        Returns the number of terminal tokens used by the encoder during batch encoding when
+        add_terminals is set to True.
+        """
+        return 0
+
+    def batch_encode(
+        self, examples: List[str], char_padding_length: int = None, char_add_terminals=True,
+        add_terminals=False, _return_tokenized_examples: bool = False, **kwargs
+    ) -> BatchData:
+
+        if add_terminals:
+            msg = f"The param 'add_terminals' must be False to encode a batch using " \
+                  f"{self.__class__.__name__}."
+            logger.error(msg)
+            raise ValueError(msg)
+
+        batch_data = super().batch_encode(
+            examples=examples, add_terminals=False, _return_tokenized_examples=True, **kwargs
+        )
+
+        # use tokenized examples to obtain tokens for char tokenization
+        _examples = batch_data.pop("_examples")
+        char_seq_ids, char_seq_lengths = [], []
+        for _seq_tokens in _examples:
+            # compute padding length for character sequences
+            _curr_max = max([len(word) for word in _seq_tokens])
+            _curr_max = _curr_max + self.number_of_char_terminal_tokens \
+                if char_add_terminals else _curr_max
+            char_padding_length = (
+                min(char_padding_length, _curr_max) if char_padding_length else _curr_max
+            )
+            _char_seq_ids, _char_seq_lengths = zip(*[
+                self._encode_chars(list(word), char_padding_length, char_add_terminals)
+                for word in _seq_tokens
+            ])
+            char_seq_ids.append(_char_seq_ids)
+            char_seq_lengths.append(_char_seq_lengths)
+
+        batch_data.update({
+            "char_seq_ids": [torch.as_tensor(_ids, dtype=torch.long) for _ids in char_seq_ids],
+            "char_seq_lengths": [torch.as_tensor(_ls, dtype=torch.long) for _ls in char_seq_lengths]
+        })
+        return batch_data
+
+    def _encode_chars(self, list_of_tokens: List[str], padding_length: int, add_terminals: bool):
+        """
+        Encodes a list of tokens in to a list of character ids based on the encoder's char vocab
+
+        Args:
+            list_of_tokens (List[str]): List of chars that are to be encoded
+            padding_length (int): Maximum length of the encoded sequence; sequences shorter than
+                this length are padded with a pad index while longer sequences are trimmed
+            add_terminals (bool): Whether terminal start and end tokens are to be added or not to
+                the encoded sequence
+
+        Returns:
+            list_of_ids (List[int]): Sequence of ids corresponding to the input chars
+            seq_length (int): The length of sequence upon encoding (before padding)
+        """
+        list_of_chars = (
+            list_of_tokens[:padding_length - self.number_of_char_terminal_tokens]
+            if add_terminals else list_of_tokens[:padding_length]
+        )
+        list_of_chars_with_terminals = (
+            [getattr(self, "char_start_token")] +
+            list_of_chars +
+            [getattr(self, "char_end_token")]
+        ) if add_terminals else list_of_chars
+        list_of_chars_with_terminals_and_padding = (
+            list_of_chars_with_terminals +
+            [getattr(self, "char_pad_token")] * (
+                padding_length - len(list_of_chars_with_terminals))
+        )
+        list_of_ids = [
+            self.char_token2id.get(token, getattr(self, "char_unk_token_idx"))
+            for token in list_of_chars_with_terminals_and_padding
+        ]
+        return list_of_ids, len(list_of_chars_with_terminals)
+
+    def get_char_vocab(self) -> Dict:
+        return self.char_token2id
+
+    def get_char_pad_token_idx(self) -> Union[None, int]:
+        """
+        If there exists a char padding token's index in the vocab, it is returned; useful while
+        initializing an embedding layer. Else returns a None.
+        """
+        if not hasattr(self, "char_pad_token_idx"):
+            return None
+        return getattr(self, "char_pad_token_idx")
+
+
+class AbstractHuggingfaceTrainableEncoder(AbstractEncoder):
+    """
+    Abstract class wrapped around AbstractEncoder that is based on Huggingface's tokenizers library
+    for creating state model.
+
+    reference:
+    https://huggingface.co/docs/tokenizers/python/latest/pipeline.html
+    """
+
+    SPECIAL_TOKENS = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.tokenizer = None
+        self.trainer = Trainer
+
+        if NO_TOKENIZERS_MODULE:
+            msg = "Must install extra [transformers] by running " \
+                  "'pip install mindmeld[transformers]'"
+            raise ImportError(msg)
+
+    def prepare(self, examples: List[str]):
+        """
+        references:
+        - Huggingface: tutorials/python/training_from_memory.html @ https://tinyurl.com/6hxrtspa
+        - https://huggingface.co/docs/tokenizers/python/latest/index.html
+        """
+
+        self._prepare_pipeline()
+        trainer = self.trainer(
+            # vocab_size=30000,
+            vocab_size=100,
+            special_tokens=self.__class__.SPECIAL_TOKENS
+        )
+        self.tokenizer.train_from_iterator(examples, trainer=trainer, length=len(examples))
+
+    def _prepare_pipeline(self):
+        self.tokenizer.normalizer = normalizers.Sequence([NFD(), Lowercase(), StripAccents()])
+        self.tokenizer.pre_tokenizer = Whitespace()
+        self.tokenizer.post_processor = TemplateProcessing(
+            single="[CLS] $A [SEP]",
+            pair="[CLS] $A [SEP] $B:1 [SEP]:1",
+            special_tokens=[
+                ("[CLS]", 1),
+                ("[SEP]", 2),
+            ],
+        )
+        self.tokenizer.enable_padding(
+            pad_id=self.__class__.SPECIAL_TOKENS.index("[PAD]"),
+            pad_token="[PAD]"
+        )
+
+    def dump(self, path: str):
+        filename = self._get_filename(path)
+        os.makedirs(os.path.dirname(filename), exist_ok=True)
+        self.tokenizer.save(filename)
+        msg = f"The state of {self.__class__.__name__} is successfully dumped at '{filename}'"
+        logger.info(msg)
+
+    def load(self, path: str):
+        filename = self._get_filename(path)
+        self.tokenizer = _get_module_or_attr("tokenizers", "Tokenizer").from_file(filename)
+        self._prepare_pipeline()
+        msg = f"The state of {self.__class__.__name__} is successfully loaded from '{filename}'"
+        logger.info(msg)
+
+    def _get_filename(self, path: str):
+        if not os.path.isdir(path):
+            msg = f"The dump method of {self.__class__.__name__} only accepts diretory as the " \
+                  f"path argument."
+            logger.error(msg)
+            raise ValueError(msg)
+        filename = os.path.join(path, "tokenizer.json")
+        return filename
+
+    def _tokenize(self, text: str) -> List[str]:
+        """
+        Example:
+        --------
+        output = tokenizer.encode("Hello, y'all! How are you 😁 ?")
+        print(output.tokens)
+        # ["Hello", ",", "y", "'", "all", "!", "How", "are", "you", "[UNK]", "?"]
+        """
+        output = self.tokenizer.encode(text, add_special_tokens=False)
+        # By disabling add_special_tokens, one can expect tokenized outputs without any terminal
+        # [CLS] and [SEP] tokens
+        return output.tokens
+
+    def batch_encode(
+        self, examples: List[str], padding_length: int = None, add_terminals=True, **kwargs
+    ) -> BatchData:
+        """
+        Example:
+        --------
+        output = tokenizer.encode_batch(["Hello, y'all!", "How are you 😁 ?"])
+        print(output[1].tokens)
+        # ["[CLS]", "How", "are", "you", "[UNK]", "?", "[SEP]", "[PAD]"]
+
+        NOTE:
+        -----
+        Passing the argument `padding_length` to set the max length for batch encoding is not
+        available yet for Huggingface tokenizers
+        """
+
+        if not add_terminals:
+            msg = f"The param 'add_terminals' must be True to encode a batch using " \
+                  f"{self.__class__.__name__}."
+            logger.error(msg)
+            raise ValueError(msg)
+
+        if padding_length is not None:
+            msg = f"{self.__class__.__name__} does not support setting padding length during" \
+                  f"batch_encode() method."
+            logger.warning(msg)
+
+        n_terminals = self.number_of_terminal_tokens if add_terminals else 0
+
+        output = self.tokenizer.encode_batch(examples, add_special_tokens=True)
+        seq_ids = [o.ids for o in output]
+        attention_masks = [o.attention_mask for o in output]
+        words = [o.words for o in output]
+
+        split_lengths = []
+        for words_nums in words:  # an eg. sequence [None,0,1,2,3,3,3,3,4,4,...,16,None,...,None]
+            curr_len = 0
+            curr_num = 0
+            _split_lengths = []
+            for word_num in words_nums[1:]:  # the first corresponds to CLS token
+                if word_num == curr_num:
+                    curr_len += 1
+                elif word_num is None:
+                    _split_lengths.append(curr_len)
+                    break
+                else:
+                    assert word_num == curr_num + 1, print(word_num, curr_num)
+                    _split_lengths.append(curr_len)
+                    curr_len = 1
+                    curr_num = word_num
+            split_lengths.append(_split_lengths)
+
+        return BatchData(**{
+            # num of groups per example
+            "seq_lengths": torch.as_tensor(  # Tensor1d[int]
+                [len(_split_lengths) + n_terminals for _split_lengths in split_lengths],
+                dtype=torch.long
+            ),
+            # len of each subgroup; for each example, sum of its split_lengths will be equal to
+            # the sum of attention mask minus terminals.
+            "split_lengths": [
+                torch.as_tensor(_split_lengths, dtype=torch.long)
+                for _split_lengths in split_lengths
+            ],  # List[Tensor1d[int]],
+            "seq_ids": torch.as_tensor(seq_ids, dtype=torch.long),  # Tensor2d[int]
+            "attention_masks": torch.as_tensor(attention_masks, dtype=torch.long),  # Tensor2d[int]
+        })
+
+    def get_vocab(self) -> Dict:
+        return self.tokenizer.get_vocab()
+
+    def get_pad_token_idx(self) -> int:
+        return self.tokenizer.token_to_id("[PAD]")
+
+
+class BytePairEncodingEncoder(AbstractHuggingfaceTrainableEncoder):
+    """
+    Encoder that fits a BPE model based on the input examples
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.tokenizer = _get_module_or_attr("tokenizers", "Tokenizer")(BPE())
+        self.trainer = BpeTrainer
+
+
+class WordPieceEncoder(AbstractHuggingfaceTrainableEncoder):
+    """
+    Encoder that fits a WordPiece model based on the input examples
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.tokenizer = _get_module_or_attr("tokenizers", "Tokenizer")(WordPiece())
+        self.trainer = WordPieceTrainer
+
+
+class HuggingfacePretrainedEncoder(AbstractEncoder):
+
+    def __init__(self, pretrained_model_name_or_path=None, **kwargs):
+        super().__init__(**kwargs)
+        self.pretrained_model_name_or_path = pretrained_model_name_or_path
+        self.config, self.tokenizer = None, None
+
+        self._number_of_terminal_tokens = None
+        self.__model_max_length = -1
+
+    def prepare(self, examples: List[str]):
+        del examples
+
+        if self.pretrained_model_name_or_path is None:
+            msg = f"Need a valid 'pretrained_model_name_or_path' path to fit " \
+                  f"{self.__class__.__name__} but found value: {self.pretrained_model_name_or_path}"
+            raise ValueError(msg)
+
+        hf_trans = HuggingfaceTransformersContainer(self.pretrained_model_name_or_path)
+        self.config = hf_trans.get_transformer_model_config()
+        self.tokenizer = hf_trans.get_transformer_model_tokenizer()
+
+    def dump(self, path: str):
+        if not os.path.isdir(path):
+            msg = f"The dump method of {self.__class__.__name__} only accepts diretory as the " \
+                  f"path argument."
+            logger.error(msg)
+            raise ValueError(msg)
+        os.makedirs(path, exist_ok=True)
+        self.config.save_pretrained(path)
+        self.tokenizer.save_pretrained(path)
+
+    def load(self, path: str):
+        if not os.path.isdir(path):
+            msg = f"The dump method of {self.__class__.__name__} only accepts diretory as the " \
+                  f"path argument."
+            logger.error(msg)
+            raise ValueError(msg)
+        hf_trans = HuggingfaceTransformersContainer(path)
+        self.config = hf_trans.get_transformer_model_config()
+        self.tokenizer = hf_trans.get_transformer_model_tokenizer()
+
+    def _tokenize(self, text: str) -> List[str]:
+        return self.tokenizer.tokenize(text)
+
+    @property
+    def number_of_terminal_tokens(self) -> int:
+        """Overwrite parent class' definition of number of terminal tokens"""
+        if not self._number_of_terminal_tokens:
+            self._number_of_terminal_tokens = len(self.tokenizer.encode(""))
+        return self._number_of_terminal_tokens
+
+    @property
+    def _model_max_length(self) -> int:
+        """Returns the maximum length of tokens per example allowed for the pretrained tokenizer"""
+        if self.__model_max_length == -1:
+            try:
+                self.__model_max_length = self.tokenizer.model_max_length
+            except AttributeError as e:
+                # case in which the huggingface tokenizer doesn't have this attribute
+                logger.info(e)
+                self.__model_max_length = None
+        return self.__model_max_length
+
+    def batch_encode(
+        self, examples: List[str], padding_length: int = None, add_terminals=True, **kwargs
+    ) -> BatchData:
+
+        if not add_terminals:
+            msg = f"The param 'add_terminals' must be True to encode a batch using " \
+                  f"{self.__class__.__name__}."
+            logger.error(msg)
+            raise ValueError(msg)
+
+        split_at = " "
+
+        n_terminals = self.number_of_terminal_tokens if add_terminals else 0
+
+        # tokenize each word of each input separately
+        # get maximum length of each example, accounting for terminal tokens- cls, sep
+        # If padding_length is None, padding has to be done to the max length of the input batch
+        tokenized_examples = [
+            [self._tokenize(word) for word in example.split(split_at)] for example in examples
+        ]
+
+        max_curr_len = max([len(sum(t_ex, [])) for t_ex in tokenized_examples]) + n_terminals
+        padding_length_including_terminals = min(max_curr_len, padding_length) \
+            if padding_length else max_curr_len
+        if self._model_max_length:
+            # padding_length cannot exceed the transformer model's maximum length
+            padding_length_including_terminals = min(
+                padding_length_including_terminals, self._model_max_length)
+        padding_length_excluding_terminals = padding_length_including_terminals - n_terminals
+
+        _trimmed_examples = [
+            _trim_a_list_of_sub_token_groups(tokenized_example, padding_length_excluding_terminals)
+            for tokenized_example in tokenized_examples
+        ]  # List[List[List[str]]], innermost List[str] is a list of sub-words for a given word
+
+        split_lengths = [[len(x) for x in ex] for ex in _trimmed_examples]
+
+        # Calling ```tokenized_examples = [" ".join(sum(ex, [])) for ex in _trimmed_examples]```
+        # & then passing it to ```self.tokenizer.__call__(tokenized_examples, ...)``` inadvertently
+        # re-tokenizes already tokenized strings. Eg: "ttyl" tokenized to ["t", "##ty", "#l"] and
+        # then modified to "t ##ty ##l" is incorrectly tokenized again in __call__ as ["t", "#",
+        # "#", "t", "y", "#", "#", "l"] and then encoded into ids.
+        # This is not the case with few children of AbstractHuggingfaceTrainableEncoder (e.g.
+        # BytePairEncodingEncoder) which has no prepending tokens such as '##' for sub-words.
+        _detokenized_examples = [
+            split_at.join(
+                [self.tokenizer.convert_tokens_to_string(group) for group in _trimmed_example]
+            )
+            for _trimmed_example in _trimmed_examples
+        ]
+        hgf_encodings = self.tokenizer(
+            _detokenized_examples, padding=True, truncation=True, max_length=None,
+            return_tensors="pt"
+        )  # Huggingface returns this as BatchEncodings object;needs to be converted to a dictionary
+
+        return BatchData(**{
+            # number of groups per example
+            "seq_lengths": torch.as_tensor(  # Tensor1d[int]
+                [len(_split_lengths) + n_terminals for _split_lengths in split_lengths],
+                dtype=torch.long
+            ),
+            # len of each subgroup; for each example, sum of its split_lengths will be equal to
+            # the sum of attention mask minus terminals.
+            "split_lengths": [
+                torch.as_tensor(_split_lengths, dtype=torch.long)
+                for _split_lengths in split_lengths
+            ],  # List[Tensor1d[int]],
+            # all the different outputs produced by huggingface's pretrained tokenizer,
+            # consisting of inputs_ids, attention_masks, etc.
+            "hgf_encodings": {**hgf_encodings},
+        })
+
+    def get_vocab(self) -> Dict:
+        return self.tokenizer.get_vocab()
+
+    def get_pad_token_idx(self) -> int:
+        return self.tokenizer.pad_token_id
+
+
+class InputEncoderFactory:
+    TOKENIZER_NAME_TO_CLASS = {
+        TokenizerType.WHITESPACE_TOKENIZER: WhitespaceEncoder,
+        TokenizerType.CHAR_TOKENIZER: CharEncoder,
+        TokenizerType.WHITESPACE_AND_CHAR_DUAL_TOKENIZER: WhitespaceAndCharDualEncoder,
+        TokenizerType.BPE_TOKENIZER: BytePairEncodingEncoder,
+        TokenizerType.WORDPIECE_TOKENIZER: WordPieceEncoder,
+        TokenizerType.HUGGINGFACE_PRETRAINED_TOKENIZER: HuggingfacePretrainedEncoder,
+    }
+
+    @classmethod
+    def get_encoder_cls(cls, tokenizer_type: str):
+        try:
+            return InputEncoderFactory.TOKENIZER_NAME_TO_CLASS[TokenizerType(tokenizer_type)]
+        except ValueError as e:
+            msg = f"Expected tokenizer_type amongst " \
+                  f"{[v.value for v in TokenizerType.__members__.values()]} " \
+                  f"but found '{tokenizer_type}'. Cannot create an input encoder."
+            logger.error(msg)
+            raise ValueError(msg) from e
diff --git a/mindmeld/models/nn_utils/layers.py b/mindmeld/models/nn_utils/layers.py
new file mode 100644
index 000000000..682c1fc87
--- /dev/null
+++ b/mindmeld/models/nn_utils/layers.py
@@ -0,0 +1,381 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2015 Cisco Systems, Inc. and others.  All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from typing import List, Union, Dict
+
+import numpy as np
+
+from .._util import _get_module_or_attr
+
+try:
+    import torch
+    import torch.nn as nn
+    import torch.nn.functional as F
+    from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence, pad_sequence
+
+    nn_module = _get_module_or_attr("torch.nn", "Module")
+except ImportError:
+    nn_module = object
+    pass
+
+logger = logging.getLogger(__name__)
+
+
+class EmbeddingLayer(nn_module):
+    """A pytorch wrapper layer for embeddings that takes input a batched sequence of ids
+    and outputs embeddings corresponding to those ids
+    """
+
+    def __init__(
+        self,
+        num_tokens: int,
+        emb_dim: int,
+        padding_idx: int = None,
+        embedding_weights: Dict[int, Union[List, np.ndarray]] = None,
+        update_embeddings: bool = True,
+        embeddings_dropout: float = 0.5,
+        coefficients: List[float] = None,
+        update_coefficients: bool = True
+    ):
+        """
+        Args:
+            num_tokens (int): size of the dictionary of embeddings
+            emb_dim (int): the size of each embedding vector
+            padding_idx (int, Optional): If given, pads the output with the embedding vector at
+                `padding_idx` (initialized to zeros) whenever it encounters the index.
+            embedding_weights (Dict[int, Union[List, np.ndarray]], Optional): weights to overwrite
+                the already initialized embedding weights
+            update_embeddings (bool, Optional): whether to freeze or train the embedding weights
+            embeddings_dropout (float, Optional): dropout rate to apply on the forward call
+            coefficients (List[float], Optional): weight coefficients for the dictionary of
+                embeddings
+            update_coefficients (bool, Optional): whether to freeze or train the coefficients
+        """
+        super().__init__()
+
+        self.embeddings = nn.Embedding(num_tokens, emb_dim, padding_idx=padding_idx)
+        if embedding_weights is not None:
+            if isinstance(embedding_weights, dict):
+                # when weights are passed as dict with keys as indices and values as embeddings
+                for idx, emb in embedding_weights.items():
+                    self.embeddings.weight.data[idx] = torch.as_tensor(emb)
+                msg = f"Initialized {len(embedding_weights)} number of embedding weights " \
+                      f"from the embedder model"
+                logger.info(msg)
+            else:
+                # when weights are passed as an array or tensor
+                self.embeddings.load_state_dict({'weight': torch.as_tensor(embedding_weights)})
+        self.embeddings.weight.requires_grad = update_embeddings
+
+        self.embedding_for_coefficients = None
+        if coefficients is not None:
+            if not len(coefficients) == num_tokens:
+                msg = f"Length of coefficients ({len(coefficients)}) must match the number of " \
+                      f"embeddings ({num_tokens})"
+                raise ValueError(msg)
+            self.embedding_for_coefficients = nn.Embedding(num_tokens, 1, padding_idx=padding_idx)
+            self.embedding_for_coefficients.load_state_dict(
+                {'weight': torch.as_tensor(coefficients).view(-1, 1)}
+            )
+            self.embedding_for_coefficients.weight.requires_grad = update_coefficients
+
+        self.dropout = nn.Dropout(embeddings_dropout)
+
+    def forward(self, padded_token_ids: "Tensor2d[int]") -> "Tensor3d[float]":
+        # padded_token_ids: dim: [BS, SEQ_LEN]
+        # returns:          dim: [BS, SEQ_LEN, EMB_DIM]
+
+        # [BS, SEQ_LEN] -> [BS, SEQ_LEN, EMB_DIM]
+        outputs = self.embeddings(padded_token_ids)
+
+        if self.embedding_for_coefficients:
+            # [BS, SEQ_LEN] -> [BS, SEQ_LEN, 1]
+            coefficients = self.embedding_for_coefficients(padded_token_ids)
+            # [BS, SEQ_LEN, EMB_DIM] -> [BS, SEQ_LEN, EMB_DIM]
+            outputs = torch.mul(outputs, coefficients)
+
+        outputs = self.dropout(outputs)
+
+        return outputs
+
+
+class CnnLayer(nn_module):
+    """A pytorch wrapper layer for 2D Convolutions
+    """
+
+    def __init__(self, emb_dim: int, kernel_sizes: List[int], num_kernels: List[int]):
+        """
+        Args:
+            emb_dim (int): the size of embedding vectors or last dimension of hidden state prior
+                to this CNN layer (i.e. width for convolution filters)
+            kernel_sizes (List[int]): the length of each kernel provided as a list of lengths
+                (i.e. length for convolution filters)
+            num_kernels (List[int]): the number of kernels for each kernel size provided as a list
+                of numbers (one number per provided size of kernel)
+        """
+        super().__init__()
+
+        if isinstance(num_kernels, list) and len(num_kernels) != len(kernel_sizes):
+            # incorrect length of num_kernels list specified
+            num_kernels = [num_kernels[0]] * len(kernel_sizes)
+        elif isinstance(num_kernels, int) and num_kernels > 0:
+            # num_kernels is a single integer value
+            num_kernels = [num_kernels] * len(kernel_sizes)
+        elif not isinstance(num_kernels, list):
+            msg = f"Invalid value for num_kernels: {num_kernels}. " \
+                  f"Expected a list of same length as emb_dim ({len(emb_dim)})"
+            raise ValueError(msg)
+
+        self.convs = nn.ModuleList()
+        # Unsqueeze input dim [BS, SEQ_LEN, EMD_DIM] to [BS, 1, SEQ_LEN, EMDDIM] and send as input
+        # Each conv module output's dimensions are [BS, n, SEQ_LEN, 1]
+        for kernel_size, num_kernel in zip(kernel_sizes, num_kernels):
+            self.convs.append(
+                nn.Sequential(
+                    nn.Conv2d(1, num_kernel, (kernel_size, emb_dim), padding=(kernel_size - 1, 0),
+                              dilation=1, bias=True, padding_mode='zeros'),
+                    nn.ReLU(),
+                )
+            )
+
+    def forward(self, padded_token_embs: "Tensor3d[float]") -> "Tensor2d[float]":
+        # padded_token_embs: dim: [BS, SEQ_LEN, EMD_DIM]
+        # returns:           dim: [BS, EMB_DIM`]
+
+        # [BS, SEQ_LEN, EMD_DIM] -> [BS, 1, SEQ_LEN, EMD_DIM]
+        embs_unsqueezed = torch.unsqueeze(padded_token_embs, dim=1)
+
+        # [BS, 1, SEQ_LEN, EMD_DIM] -> list([BS, n, SEQ_LEN])
+        conv_outputs = [conv(embs_unsqueezed).squeeze(3) for conv in self.convs]
+
+        # list([BS, n, SEQ_LEN]) -> list([BS, n])
+        maxpool_conv_outputs = [F.max_pool1d(out, out.size(2)).squeeze(2) for out in conv_outputs]
+
+        # list([BS, n]) -> [BS, sum(n)]
+        outputs = torch.cat(maxpool_conv_outputs, dim=1)
+
+        return outputs
+
+
+class LstmLayer(nn_module):
+    """A pytorch wrapper layer for BiLSTMs
+    """
+
+    def __init__(
+        self,
+        emb_dim: int,
+        hidden_dim: int,
+        num_layers: int,
+        lstm_dropout: float,
+        bidirectional: bool
+    ):
+        """
+        Args:
+            emb_dim (int): the size of embedding vectors or last dimension of hidden state prior
+                to this LSTM layer
+            hidden_dim (int): the hidden dimension for nn.LSTM
+            num_layers (int): the number of nn.LSTM layers to stack
+            lstm_dropout (float): the dropout rate for nn.LSTM
+            bidirectional (bool): whether LSTMs should be applied on both forward and
+                backward sequences of the input or not
+        """
+        super().__init__()
+
+        self.lstm = nn.LSTM(
+            emb_dim, hidden_dim, num_layers=num_layers, dropout=lstm_dropout,
+            bidirectional=bidirectional, batch_first=True
+        )
+
+    def forward(
+        self,
+        padded_token_embs: "Tensor3d[float]",
+        lengths: "Tensor1d[int]",
+    ) -> "Tensor3d[float]":
+        # padded_token_embs: dim: [BS, SEQ_LEN, EMD_DIM]
+        # lengths:           dim: [BS]
+        # returns:           dim: [BS, SEQ_LEN, EMB_DIM]
+
+        # [BS, SEQ_LEN, EMD_DIM] -> [BS, SEQ_LEN, EMD_DIM*(2 if bidirectional else 1)]
+        lengths = lengths.to(torch.device("cpu"))
+        packed = pack_padded_sequence(padded_token_embs, lengths,
+                                      batch_first=True, enforce_sorted=False)
+        lstm_outputs, _ = self.lstm(packed)
+        outputs = pad_packed_sequence(lstm_outputs, batch_first=True)[0]
+
+        return outputs
+
+
+class PoolingLayer(nn_module):
+    """A pooling layer for Tensor3d objects that pools along the last dimension. Assumes that
+    padding if any exists on the right side of inputs (i.e. not in the beginning of inputs)
+    """
+
+    def __init__(self, pooling_type: str):
+        """
+        Args:
+            pooling_type (str): the choice of pooling; to be amongst following:
+                first: the first index of each sequence will be the pooled output (similar to CLS
+                    token in BERT models)
+                last: the last index of each sequence will be the pooled output (useful for pooling
+                    outputs from nn.LSTM)
+                max: max pool across last dimension will be the pooled output
+                mean: mean pool across last dimension will be the pooled output
+                mean_sqrt: similar to 'mean' but slashed by the square root of sequence length
+        """
+        super().__init__()
+
+        pooling_type = pooling_type.lower()
+
+        allowed_pooling_types = ["first", "last", "max", "mean", "mean_sqrt"]
+        if pooling_type not in allowed_pooling_types:
+            msg = f"Expected pooling_type amongst {allowed_pooling_types} " \
+                  f"but found '{pooling_type}'"
+            raise ValueError(msg)
+
+        # assumption: first token is never a pad token for the passed inputs
+        self._requires_length = ["last", "max", "mean", "mean_sqrt"]
+
+        self.pooling_type = pooling_type
+
+    def forward(
+        self,
+        padded_token_embs: "Tensor3d[float]",
+        lengths: "Tensor1d[int]" = None,
+    ) -> "Tensor2d[float]":
+        # padded_token_embs: dim: [BS, SEQ_LEN, EMD_DIM]
+        # lengths:           dim: [BS]
+        # returns:           dim: [BS, EMD_DIM]
+
+        if self.pooling_type in self._requires_length and lengths is None:
+            msg = f"Missing required value 'lengths' for pooling_type: {self.pooling_type}"
+            raise ValueError(msg)
+
+        if self.pooling_type == "first":
+            outputs = padded_token_embs[:, 0, :]
+        elif self.pooling_type == "last":
+            last_seq_idxs = torch.LongTensor([x - 1 for x in lengths])
+            outputs = padded_token_embs[range(padded_token_embs.shape[0]), last_seq_idxs, :]
+        else:
+            mask = pad_sequence(
+                [torch.as_tensor([1] * length_) for length_ in lengths],
+                batch_first=True,
+                padding_value=0.0,
+            ).unsqueeze(-1).expand(padded_token_embs.size()).float().to(padded_token_embs.device)
+            if self.pooling_type == "max":
+                padded_token_embs[mask == 0] = -1e9  # set to a large negative value
+                outputs, _ = torch.max(padded_token_embs, dim=1)
+            elif self.pooling_type == "mean":
+                summed_padded_token_embs = torch.sum(padded_token_embs * mask, dim=1)
+                outputs = summed_padded_token_embs / mask.sum(1)
+            elif self.pooling_type == "mean_sqrt":
+                summed_padded_token_embs = torch.sum(padded_token_embs * mask, dim=1)
+                expanded_lengths = lengths.unsqueeze(dim=1).expand(summed_padded_token_embs.size())
+                outputs = torch.div(summed_padded_token_embs, torch.sqrt(expanded_lengths.float()))
+
+        return outputs
+
+
+class SplittingAndPoolingLayer(nn_module):
+    """Pooling class that first splits a sequence of representations into subgroups of
+    representations based on lengths of subgroups inputted, and pools each subgroup separately.
+    """
+
+    def __init__(self, pooling_type: str, number_of_terminal_tokens: int):
+        """
+        Args:
+            pooling_type (str): the choice of pooling; to be amongst following:
+                first: the first index of each subsequence will be the pooled output of that
+                    subgroup(e.g. token classification using BERT models with sub-word tokenization)
+                last: the last index of each subsequence will be the pooled output of that subgroup
+                    (e.g. for word level representations when using a character BiLSTM)
+                max: max pool across subsequence will be the pooled output of that subgroup
+                mean: mean pool across subsequence will be the pooled output of that subgroup
+                    (e.g. token classification using BERT models with sub-word tokenization)
+                mean_sqrt: similar to 'mean' but slashed by the square root of subsequence length
+            number_of_terminal_tokens (int): the number of terminal tokens that will be discarded
+                if discard terminals is set to True in the forward method.
+        """
+        super().__init__()
+
+        self.pooling_type = pooling_type.lower()
+        self.number_of_terminal_tokens = number_of_terminal_tokens
+
+        self.pooling_layer = PoolingLayer(pooling_type=self.pooling_type)
+
+    def _split_and_pool(
+        self,
+        tensor_2d: "Tensor2d[float]",
+        list_of_subgroup_lengths: "Tensor1d[int]",
+        discard_terminals: bool
+    ):
+        # tensor_2d:                 dim: [SEQ_LEN, EMD_DIM]
+        # list_of_subgroup_lengths:  dim: List of int summing up to SEQ_LEN' <= SEQ_LEN
+        # discard_terminals:         bool
+        # returns:                   dim: [SEQ_LEN``, EMD_DIM]
+
+        if discard_terminals:
+            # TODO: Number of terminals can also be 1 (maybe just left or just right) in some models
+            if self.number_of_terminal_tokens != 2:
+                msg = f"Unable to combine sub-tokens' representations for each word into one in " \
+                      f"{self.__class__.__name__}. It is possible that your choice of tokenizer " \
+                      f"has {self.number_of_terminal_tokens} terminal token instead of assumed " \
+                      f"2 terminals."  # (eg. t5-base tokenizer)
+                raise NotImplementedError(msg)
+
+            # since list_of_subgroup_lengths consists of lengths of only non-terminal subgroups but
+            # the inputted tensor_2d consists of terminals
+            seq_len_required = sum(list_of_subgroup_lengths) + self.number_of_terminal_tokens
+            tensor_2d_with_terminals = tensor_2d[:seq_len_required]
+            tensor_2d = tensor_2d_with_terminals[1:-1]  # discard terminal representations
+        else:
+            seq_len_required = sum(list_of_subgroup_lengths)
+            tensor_2d = tensor_2d[:seq_len_required]
+
+        try:
+            # argument 'split_sizes' (position 1) must be tuple of ints, not Tensor
+            splits = torch.split(tensor_2d, list_of_subgroup_lengths.tolist(), dim=0)
+        except RuntimeError as e:
+            msg = f"Unable to combine sub-tokens' representations for each word into one in " \
+                  f"{self.__class__.__name__}. It is possible that your choice of tokenizer " \
+                  f"does not split input text at whitespace (eg. robert-base tokenizer), due " \
+                  f"to which one-representation-per-word cannot be obtained to do tagging at " \
+                  f"word-level for token classification."
+            raise ValueError(msg) from e
+        padded_token_embs = pad_sequence(splits, batch_first=True)  # [BS', SEQ_LEN', EMD_DIM]
+
+        # return dims: [len(list_of_subgroup_lengths), EMD_DIM]
+        pooled_repr_for_each_subgroup = self.pooling_layer(
+            padded_token_embs=padded_token_embs,
+            lengths=list_of_subgroup_lengths
+        )
+
+        return pooled_repr_for_each_subgroup
+
+    def forward(
+        self,
+        padded_token_embs: "Tensor3d[float]",
+        span_lengths: "List[Tensor1d[int]]",
+        discard_terminals: bool = None
+    ):
+        # padded_token_embs: dim: [BS, SEQ_LEN, EMD_DIM]
+        # span_lengths:      dim: List[List of int summing up to SEQ_LEN' <= SEQ_LEN]
+        # discard_terminals: bool
+        # returns:           dim: [BS, SEQ_LEN', EMD_DIM]
+
+        outputs = pad_sequence([
+            self._split_and_pool(_padded_token_embs, _span_lengths, discard_terminals)
+            for _padded_token_embs, _span_lengths in zip(padded_token_embs, span_lengths)
+        ], batch_first=True)
+
+        return outputs
diff --git a/mindmeld/models/nn_utils/sequence_classification.py b/mindmeld/models/nn_utils/sequence_classification.py
new file mode 100644
index 000000000..7b152c594
--- /dev/null
+++ b/mindmeld/models/nn_utils/sequence_classification.py
@@ -0,0 +1,487 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2015 Cisco Systems, Inc. and others.  All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Custom modules built on top of nn layers that can do sequence classification
+"""
+
+import logging
+from abc import abstractmethod
+from collections import OrderedDict
+from typing import List
+
+from .classification import BaseClassification
+from .helpers import BatchData, EmbedderType, SequenceClassificationType
+from .layers import (
+    EmbeddingLayer,
+    CnnLayer,
+    LstmLayer,
+    PoolingLayer
+)
+from ..containers import HuggingfaceTransformersContainer
+
+try:
+    import torch
+    import torch.nn as nn
+    import torch.nn.functional as F
+except ImportError:
+    pass
+
+logger = logging.getLogger(__name__)
+
+
+class BaseSequenceClassification(BaseClassification):
+    """Base class that defines all the necessary elements to successfully train/infer
+     custom pytorch modules wrapped on top of this base class. Classes derived from
+     this base can be trained for sequence classification.
+    """
+
+    def _prepare_labels(self, labels: List[int], max_length: int = None):
+        # for sequence classification, the length of an example doesn't matter as we have only one
+        # label for each example. hence, no need to do any padding or validation checks.
+        del max_length
+        return torch.as_tensor(labels, dtype=torch.long)
+
+    def _init_graph(self):
+
+        # initialize the beginning layers of the graph
+        self._init_core()
+
+        # init the underlying params and architectural components
+        try:
+            assert self.out_dim > 0
+            self.params.update({"out_dim": self.out_dim})
+        except (AttributeError, AssertionError) as e:
+            msg = f"Derived class '{self.name}' must indicate its hidden size for dense layer " \
+                  f"classification by having an attribute 'self.out_dim', which must be a " \
+                  f"positive integer greater than 1"
+            raise ValueError(msg) from e
+
+        # init the peripheral architecture params
+        if not self.params.num_labels:
+            msg = f"Invalid number of labels ({self.params.num_labels}) inputted to '{self.name}'"
+            raise ValueError(msg)
+
+        # init the peripheral architectural components and the criterion to compute loss
+        self.dense_layer_dropout = nn.Dropout(
+            p=1 - self.params.output_keep_prob
+        )
+        if self.params.num_labels == 2:
+            # sigmoid criterion
+            self.classifier_head = nn.Linear(self.out_dim, 1)
+            self.criterion = nn.BCEWithLogitsLoss(reduction='mean')
+        elif self.params.num_labels > 2:
+            # cross-entropy criterion
+            self.classifier_head = nn.Linear(self.out_dim, self.params.num_labels)
+            self.criterion = nn.CrossEntropyLoss(reduction='mean')
+        else:
+            msg = f"Invalid number of labels specified: {self.params.num_labels}. " \
+                  f"A valid number is equal to or greater than 2"
+            raise ValueError(msg)
+
+        msg = f"{self.name} is initialized"
+        logger.info(msg)
+
+    def forward(self, batch_data):
+
+        batch_data = self.to_device(batch_data)
+        batch_data = self._forward_core(batch_data)
+
+        seq_embs = batch_data["seq_embs"]
+        seq_embs = self.dense_layer_dropout(seq_embs)
+        logits = self.classifier_head(seq_embs)
+        batch_data.update({"logits": logits})
+
+        targets = batch_data.pop("_labels", None)
+        if targets is not None:
+            if self.params.num_labels == 2:
+                loss = self.criterion(logits.view(-1), targets.float())
+            else:  # self.params.num_labels > 2:
+                loss = self.criterion(logits, targets)
+            batch_data.update({"loss": loss})
+
+        return batch_data
+
+    def predict(self, examples):
+        logits = self._forward_with_batching_and_no_grad(examples)
+        if self.params.num_labels == 2:
+            predictions = (logits >= 0.5).long().view(-1)
+        else:  # self.params.num_labels > 2:
+            predictions = torch.argmax(logits, dim=-1)
+        return predictions.tolist()
+
+    def predict_proba(self, examples):
+        logits = self._forward_with_batching_and_no_grad(examples)
+        if self.params.num_labels == 2:
+            probs = F.sigmoid(logits)
+            # extending the results from shape [N,1] to [N,2] to give out class probs distinctly
+            probs = torch.cat((1 - probs, probs), dim=-1)
+        else:  # self.params.num_labels > 2:
+            probs = F.softmax(logits, dim=-1)
+        return probs.tolist()
+
+    def _forward_with_batching_and_no_grad(self, examples):
+        logits = None
+        was_training = self.training
+        self.eval()
+        with torch.no_grad():
+            for start_idx in range(0, len(examples), self.params.batch_size):
+                batch_examples = examples[start_idx:start_idx + self.params.batch_size]
+                batch_data = self.encoder.batch_encode(
+                    batch_examples,
+                    padding_length=self.params.padding_length,
+                    add_terminals=self.params.add_terminals,
+                )
+                batch_logits = self.forward(batch_data)["logits"]
+                logits = torch.cat((logits, batch_logits)) if logits is not None else batch_logits
+        if was_training:
+            self.train()
+        return logits
+
+    @abstractmethod
+    def _init_core(self) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def _forward_core(self, batch_data: BatchData) -> BatchData:
+        raise NotImplementedError
+
+
+class EmbedderForSequenceClassification(BaseSequenceClassification):
+    """An embedder pooling module that operates on a batched sequence of token ids. The
+    tokens could be characters or words or sub-words. This module finally outputs one 1D
+    representation for each instance in the batch (i.e. [BS, EMB_DIM]).
+
+    The `forward` method of this module expects padded token ids along with numer of tokens
+    per instance in the batch.
+
+    Additionally, one can set different coefficients for different tokens of the embedding
+    matrix (e.g. tf-idf weights).
+    """
+
+    def _init_core(self):
+        self.emb_layer = EmbeddingLayer(
+            self.params._num_tokens,
+            self.params.emb_dim,
+            self.params._padding_idx,
+            self.params.pop("_embedding_weights", None),
+            self.params.update_embeddings,
+            1 - self.params.embedder_output_keep_prob
+        )
+        self.emb_layer_pooling = PoolingLayer(
+            self.params.embedder_output_pooling_type
+        )
+        self.out_dim = self.params.emb_dim
+
+    def _forward_core(self, batch_data):
+        seq_ids = batch_data["seq_ids"]  # [BS, SEQ_LEN]
+
+        flattened_split_lengths = [
+            sum(_split_lengths) +
+            (self.encoder.number_of_terminal_tokens if self.params.add_terminals else 0)
+            for _split_lengths in batch_data["split_lengths"]
+        ]
+        flattened_split_lengths = torch.as_tensor(flattened_split_lengths, dtype=torch.long)  # [BS]
+
+        encodings = self.emb_layer(seq_ids)  # [BS, SEQ_LEN, EMD_DIM]
+        encodings = self.emb_layer_pooling(encodings, flattened_split_lengths)  # [BS, self.out_dim]
+
+        batch_data.update({"seq_embs": encodings})
+
+        return batch_data
+
+
+class CnnForSequenceClassification(BaseSequenceClassification):
+    """A CNN module that operates on a batched sequence of token ids. The tokens could be
+    characters or words or sub-words. This module finally outputs one 1D representation
+    for each instance in the batch (i.e. [BS, EMB_DIM]).
+
+    The `forward` method of this module expects only padded token ids as input.
+    """
+
+    def _init_core(self):
+        self.emb_layer = EmbeddingLayer(
+            self.params._num_tokens,
+            self.params.emb_dim,
+            self.params._padding_idx,
+            self.params.pop("_embedding_weights", None),
+            self.params.update_embeddings,
+            1 - self.params.embedder_output_keep_prob
+        )
+        self.conv_layer = CnnLayer(
+            self.params.emb_dim,
+            self.params.window_sizes,
+            self.params.number_of_windows
+        )
+        self.out_dim = sum(self.params.number_of_windows)
+
+    def _forward_core(self, batch_data):
+        seq_ids = batch_data["seq_ids"]  # [BS, SEQ_LEN]
+
+        encodings = self.emb_layer(seq_ids)  # [BS, SEQ_LEN, EMD_DIM]
+        encodings = self.conv_layer(encodings)  # [BS, self.out_dim]
+
+        batch_data.update({"seq_embs": encodings})
+
+        return batch_data
+
+
+class LstmForSequenceClassification(BaseSequenceClassification):
+    """A LSTM module that operates on a batched sequence of token ids. The tokens could be
+    characters or words or sub-words. This module finally outputs one 1D representation
+    for each instance in the batch (i.e. [BS, EMB_DIM]).
+
+    The `forward` method of this module expects padded token ids along with numer of tokens
+    per instance in the batch.
+    """
+
+    def _init_core(self):
+        self.emb_layer = EmbeddingLayer(
+            self.params._num_tokens,
+            self.params.emb_dim,
+            self.params._padding_idx,
+            self.params.pop("_embedding_weights", None),
+            self.params.update_embeddings,
+            1 - self.params.embedder_output_keep_prob
+        )
+        self.lstm_layer = LstmLayer(
+            self.params.emb_dim,
+            self.params.lstm_hidden_dim,
+            self.params.lstm_num_layers,
+            1 - self.params.lstm_keep_prob,
+            self.params.lstm_bidirectional
+        )
+        self.lstm_layer_pooling = PoolingLayer(
+            self.params.lstm_output_pooling_type
+        )
+        self.out_dim = (
+            self.params.lstm_hidden_dim * 2 if self.params.lstm_bidirectional
+            else self.params.lstm_hidden_dim
+        )
+
+    def _forward_core(self, batch_data):
+        seq_ids = batch_data["seq_ids"]  # [BS, SEQ_LEN]
+
+        flattened_split_lengths = [
+            sum(_split_lengths) +
+            (self.encoder.number_of_terminal_tokens if self.params.add_terminals else 0)
+            for _split_lengths in batch_data["split_lengths"]
+        ]
+        flattened_split_lengths = torch.as_tensor(flattened_split_lengths, dtype=torch.long)  # [BS]
+
+        encodings = self.emb_layer(seq_ids)  # [BS, SEQ_LEN, EMD_DIM]
+        encodings = self.lstm_layer(encodings, flattened_split_lengths)  # [BS,SEQ_LEN,self.out_dim]
+        encodings = self.lstm_layer_pooling(encodings, flattened_split_lengths)  # [BS,self.out_dim]
+
+        batch_data.update({"seq_embs": encodings})
+
+        return batch_data
+
+
+class BertForSequenceClassification(BaseSequenceClassification):
+
+    def fit(self, examples, labels, **params):
+        # overriding base class' method to set params, and then calling base class' .fit()
+
+        embedder_type = params.get("embedder_type", EmbedderType.BERT.value)
+        if EmbedderType(embedder_type) != EmbedderType.BERT:
+            msg = f"{self.name} can only be used with 'embedder_type': " \
+                  f"'{EmbedderType.BERT.value}'. " \
+                  f"Other values passed through config params are not allowed."
+            raise ValueError(msg)
+
+        safe_values = {
+            "num_warmup_steps": 50,
+            "learning_rate": 2e-5,
+            "optimizer": "AdamW",
+            "max_grad_norm": 1.0,
+        }
+
+        for k, v in safe_values.items():
+            v_inputted = params.get(k, v)
+            if v != v_inputted:
+                msg = f"{self.name} can be best used with '{k}' equal to '{v}' but found " \
+                      f"the value '{v_inputted}'. Use the non-default value with caution as it " \
+                      f"may lead to unexpected results and longer training times depending on " \
+                      f"the choice of pretrained model."
+                logger.warning(msg)
+            else:
+                params.update({k: v})
+
+        params.update({
+            "embedder_type": embedder_type,
+            "save_frozen_bert_weights": params.get("save_frozen_bert_weights", False)  # if True,
+            # frozen set of bert weights are also dumped
+        })
+
+        super().fit(examples, labels, **params)
+
+    def _create_optimizer_and_scheduler(self, num_training_steps):
+
+        num_warmup_steps = min(int(0.1 * num_training_steps), self.params.num_warmup_steps)
+        self.params.update({"num_warmup_steps": num_warmup_steps})
+
+        # https://github.com/huggingface/transformers/blob/master/src/transformers/optimization.py
+        # refer `get_linear_schedule_with_warmup` method
+        def lr_lambda(current_step: int):
+            if current_step < num_warmup_steps:
+                return float(current_step) / float(max(1, num_warmup_steps))
+            return max(
+                0.0, float(num_training_steps - current_step) / float(
+                    max(1, num_training_steps - num_warmup_steps))
+            )
+
+        # load a torch optimizer
+        optimizer = self._create_optimizer()
+        # load a lr scheduler
+        scheduler = getattr(torch.optim.lr_scheduler, "LambdaLR")(optimizer, lr_lambda)
+        return optimizer, scheduler
+
+    def _create_optimizer(self):
+        params = list(self.named_parameters())
+        no_decay = ["bias", 'LayerNorm.bias', "LayerNorm.weight",
+                    'layer_norm.bias', 'layer_norm.weight']
+        optimizer_grouped_parameters = [
+            {'params': [p for n, p in params if not any(nd in n for nd in no_decay)],
+             'weight_decay': 0.01},
+            {'params': [p for n, p in params if any(nd in n for nd in no_decay)],
+             'weight_decay': 0.0}
+        ]
+        optimizer = getattr(torch.optim, self.params.optimizer)(
+            optimizer_grouped_parameters,
+            lr=self.params.learning_rate,
+            eps=1e-08,
+            weight_decay=0.01
+        )
+        return optimizer
+
+    def _get_dumpable_state_dict(self):
+        if not self.params.update_embeddings and not self.params.save_frozen_bert_weights:
+            state_dict = OrderedDict(
+                {k: v for k, v in self.state_dict().items() if not k.startswith("bert_model")}
+            )
+            return state_dict
+        return self.state_dict()
+
+    def _init_core(self):
+
+        self.bert_model = HuggingfaceTransformersContainer(
+            self.params.pretrained_model_name_or_path,
+            cache_lookup=False
+        ).get_transformer_model()
+        if not self.params.update_embeddings:
+            for param in self.bert_model.parameters():
+                param.requires_grad = False
+
+        self.dropout = nn.Dropout(
+            p=1 - self.params.embedder_output_keep_prob
+        )
+        if self.params.embedder_output_pooling_type != "first":
+            self.emb_layer_pooling = PoolingLayer(
+                self.params.embedder_output_pooling_type
+            )
+        self.out_dim = self.params.emb_dim
+
+    def _forward_core(self, batch_data):
+
+        # refer to https://huggingface.co/docs/transformers/master/en/main_classes/output
+        # for more details on huggingface's bert outputs
+
+        bert_outputs = self.bert_model(**batch_data["hgf_encodings"], return_dict=True)
+
+        if self.params.embedder_output_pooling_type != "first":
+            # 'last_hidden_state' refers to the tensor output of the final transformer layer (i.e.
+            # before the logit layer) and hence its dimension  [BS, SEQ_LEN, EMD_DIM]
+            last_hidden_state = bert_outputs.get("last_hidden_state")  # [BS, SEQ_LEN, EMD_DIM]
+            if last_hidden_state is None:
+                # TODO: Do all huggingface models have this key? Are there any alternatives for this
+                #  key? If so, we can enumerate the set of key names instead of just one key name?
+                msg = f"The choice of pretrained bert model " \
+                      f"({self.params.pretrained_model_name_or_path}) " \
+                      f"has no key 'last_hidden_state' in its output dictionary."
+                raise ValueError(msg)
+            last_hidden_state = self.dropout(last_hidden_state)
+            flattened_split_lengths = [
+                sum(_split_lengths) +
+                (self.encoder.number_of_terminal_tokens if self.params.add_terminals else 0)
+                for _split_lengths in batch_data["split_lengths"]
+            ]
+            flattened_split_lengths = torch.as_tensor(flattened_split_lengths, dtype=torch.long)
+            encodings = self.emb_layer_pooling(
+                last_hidden_state, flattened_split_lengths)  # [BS, self.out_dim]
+        else:
+            # 'pooler_output' refers to the first token's (aka. CLS) representation obtained form
+            # the last hidden layer, and hence its dimension [BS, self.out_dim]
+            pooler_output = bert_outputs.get("pooler_output")  # [BS, self.out_dim]
+            if pooler_output is None:
+                msg = f"The chosen pretrained bert ({self.params.pretrained_model_name_or_path}) " \
+                      "has no key 'pooler_output' in its output dictionary (maybe the selected " \
+                      "choice of transformers model has no CLS kind-of token?). " \
+                      "Continuing to obtain the first representation of the last hidden state " \
+                      "due to the passed-in value for embedder output pooling type param: " \
+                      f"{self.params.embedder_output_pooling_type}. " \
+                      f"If you wish to pool differently (eg. mean pooling), change the param " \
+                      f"value in configs and run training."
+                logger.error(msg)
+                last_hidden_state = bert_outputs.get("last_hidden_state")  # [BS, SEQ_LEN, EMD_DIM]
+                if last_hidden_state is None:
+                    msg = f"The choice of pretrained bert model " \
+                          f"({self.params.pretrained_model_name_or_path}) " \
+                          f"has no key 'last_hidden_state' in its output dictionary."
+                    raise ValueError(msg)
+                pooler_output = last_hidden_state[:, 0, :]  # [BS, self.out_dim]
+            encodings = self.dropout(pooler_output)
+
+        batch_data.update({"seq_embs": encodings})
+
+        return batch_data
+
+
+def get_sequence_classifier_cls(classifier_type: str, embedder_type: str = None):
+    try:
+        classifier_type = SequenceClassificationType(classifier_type)
+    except ValueError as e:
+        msg = f"Neural Nets' sequence classification module expects classifier_type to be amongst" \
+              f" {[v.value for v in SequenceClassificationType.__members__.values()]}" \
+              f" but found '{classifier_type}'."
+        raise ValueError(msg) from e
+
+    try:
+        embedder_type = EmbedderType(embedder_type)
+    except ValueError as e:
+        msg = f"Neural Nets' sequence classification module expects embedder_type to be amongst " \
+              f" {[v.value for v in EmbedderType.__members__.values()]} " \
+              f" but found '{embedder_type}'."
+        raise ValueError(msg) from e
+
+    if (
+        embedder_type == EmbedderType.BERT and
+        classifier_type not in [SequenceClassificationType.EMBEDDER]
+    ):
+        msg = f"To use the embedder_type '{EmbedderType.BERT.value}', " \
+              f"classifier_type must be '{SequenceClassificationType.EMBEDDER.value}'."
+        raise ValueError(msg)
+
+    # disambiguation between glove, bert and non-pretrained embedders
+    def _resolve_and_return_embedder_class(_embedder_type):
+        return {
+            EmbedderType.NONE: EmbedderForSequenceClassification,
+            EmbedderType.GLOVE: EmbedderForSequenceClassification,
+            EmbedderType.BERT: BertForSequenceClassification
+        }[_embedder_type]
+
+    return {
+        SequenceClassificationType.EMBEDDER: _resolve_and_return_embedder_class(embedder_type),
+        SequenceClassificationType.CNN: CnnForSequenceClassification,
+        SequenceClassificationType.LSTM: LstmForSequenceClassification,
+    }[classifier_type]
diff --git a/mindmeld/models/nn_utils/token_classification.py b/mindmeld/models/nn_utils/token_classification.py
new file mode 100644
index 000000000..ece9496d4
--- /dev/null
+++ b/mindmeld/models/nn_utils/token_classification.py
@@ -0,0 +1,656 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2015 Cisco Systems, Inc. and others.  All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Custom modules built on top of nn layers that can do token classification
+"""
+
+import logging
+from abc import abstractmethod
+from collections import OrderedDict
+from typing import List
+
+from .classification import BaseClassification
+from .helpers import TokenClassificationType
+from .helpers import TokenizerType, EmbedderType
+from .layers import (
+    EmbeddingLayer,
+    CnnLayer,
+    LstmLayer,
+    PoolingLayer,
+    SplittingAndPoolingLayer
+)
+from ..containers import HuggingfaceTransformersContainer
+
+try:
+    import torch
+    import torch.nn as nn
+    from torch.nn.utils.rnn import pad_sequence
+except ImportError:
+    pass
+
+try:
+    from torchcrf import CRF
+
+    TORCHCRF_AVAILABLE = True
+except ImportError:
+    TORCHCRF_AVAILABLE = False
+    pass
+
+logger = logging.getLogger(__name__)
+
+
+class BaseTokenClassification(BaseClassification):
+    """Base class that defines all the necessary elements to successfully train/infer
+     custom pytorch modules wrapped on top of this base class. Classes derived from
+     this base can be trained for sequence tagging aka. token classification.
+    """
+
+    def _prepare_labels(self, labels: List[List[int]], max_length: int):
+        # for token classification, the length of an example matters (i.e. the number of
+        # (sub-)words in it) as the labels/targets are one-to-one mapped per (sub-)word.
+        # hence, we need to do padding with a _label_padding_idx
+
+        def _trim_or_pad(label_sequence):
+            if len(label_sequence) > max_length:
+                return label_sequence[:max_length]
+            else:
+                return (
+                    label_sequence + [self.params._label_padding_idx] *
+                    (max_length - len(label_sequence))
+                )
+
+        return torch.as_tensor(
+            [_trim_or_pad(_label_sequence) for _label_sequence in labels],
+            dtype=torch.long
+        )
+
+    def _init_graph(self):
+
+        self._init_core()
+
+        # init the underlying params and architectural components
+        try:
+            assert self.out_dim > 0
+            self.params.update({"out_dim": self.out_dim})
+        except (AttributeError, AssertionError) as e:
+            msg = f"Derived class '{self.name}' must indicate its hidden size for dense layer " \
+                  f"classification by having an attribute 'self.out_dim', which must be a " \
+                  f"positive integer greater than 1"
+            raise ValueError(msg) from e
+
+        # init the peripheral architecture params and architectural components
+        self.span_pooling_layer = SplittingAndPoolingLayer(
+            self.params.token_spans_pooling_type,
+            self.encoder.number_of_terminal_tokens,  # arg max number of terminal tokens
+        )
+        if not self.params.num_labels:
+            msg = f"Invalid number of labels ({self.params.num_labels}) inputted to '{self.name}'"
+            raise ValueError(msg)
+
+        # init the peripheral architectural components and the criterion to compute loss
+        self.dense_layer_dropout = nn.Dropout(
+            p=1 - self.params.output_keep_prob
+        )
+        if self.params.use_crf_layer:
+            self.classifier_head = nn.Linear(self.out_dim, self.params.num_labels)
+            if not TORCHCRF_AVAILABLE:
+                raise ImportError("pip install torchcrf")
+            self.crf_layer = CRF(self.params.num_labels, batch_first=True)
+        else:
+            if self.params.num_labels >= 2:
+                # cross-entropy criterion
+                self.classifier_head = nn.Linear(self.out_dim, self.params.num_labels)
+                self.criterion = nn.CrossEntropyLoss(
+                    reduction='mean', ignore_index=self.params._label_padding_idx)
+            else:
+                msg = f"Invalid number of labels specified: {self.params.num_labels}. " \
+                      f"A valid number is equal to or greater than 2"
+                raise ValueError(msg)
+
+        msg = f"{self.name} is initialized"
+        logger.info(msg)
+
+    def forward(self, batch_data):
+
+        batch_data = self.to_device(batch_data)
+        batch_data = self._forward_core(batch_data)
+
+        token_embs = batch_data["token_embs"]
+
+        # strip the terminals if required; note they are not accounted in the label padding process
+        token_embs = self.span_pooling_layer(
+            token_embs,
+            batch_data["split_lengths"],  # List[Tensor1d[int]],
+            discard_terminals=self.params.add_terminals
+        )  # [BS, SEQ_LEN`, EMD_DIM], and the SEQ_LEN` matches the width of labels upon padding
+
+        token_embs = self.dense_layer_dropout(token_embs)
+        logits = self.classifier_head(token_embs)
+        batch_data.update({"logits": logits})
+
+        targets = batch_data.pop("_labels", None)
+        if targets is not None:
+            if self.params.use_crf_layer:
+                # create a mask to ignore token positions that are padded
+                mask = torch.as_tensor(targets != self.params._label_padding_idx,
+                                       dtype=torch.uint8).to(self.params.device)
+                loss = - self.crf_layer(logits, targets, mask=mask)  # negative log likelihood
+            else:
+                loss = self.criterion(logits.view(-1, logits.shape[-1]), targets.view(-1))
+            batch_data.update({"loss": loss})
+
+        return batch_data
+
+    def predict(self, examples):
+        predictions = []
+        was_training = self.training
+        self.eval()
+        with torch.no_grad():
+            for start_idx in range(0, len(examples), self.params.batch_size):
+                batch_examples = examples[start_idx:start_idx + self.params.batch_size]
+                batch_data = self.encoder.batch_encode(
+                    batch_examples,
+                    padding_length=self.params.padding_length,
+                    add_terminals=self.params.add_terminals,
+                )
+                batch_logits = self.forward(batch_data)["logits"]
+                # find predictions
+                if self.params.use_crf_layer:
+                    # create a mask to ignore token positions that are padded
+                    mask = pad_sequence(
+                        [torch.as_tensor([1] * len(_split_lengths))
+                         for _split_lengths in batch_data["split_lengths"]],
+                        batch_first=True
+                    )
+                    # [BS] -> [BS, SEQ_LEN]
+                    mask = torch.as_tensor(mask, dtype=torch.uint8).to(self.params.device)
+                    batch_predictions = self.crf_layer.decode(batch_logits, mask=mask)
+                else:
+                    batch_predictions = torch.argmax(batch_logits, dim=-1).tolist()
+                # trim predictions as per sequence length
+                batch_predictions = [
+                    predictions[:len(_split_lengths)] for predictions, _split_lengths in
+                    zip(batch_predictions, batch_data["split_lengths"])
+                ]
+                predictions.extend(batch_predictions)
+        if was_training:
+            self.train()
+        return predictions
+
+    def predict_proba(self, examples):
+        prediction_tuples = []
+        was_training = self.training
+        self.eval()
+        with torch.no_grad():
+            for start_idx in range(0, len(examples), self.params.batch_size):
+                batch_examples = examples[start_idx:start_idx + self.params.batch_size]
+                batch_data = self.encoder.batch_encode(
+                    batch_examples,
+                    padding_length=self.params.padding_length,
+                    add_terminals=self.params.add_terminals,
+                )
+                batch_logits = self.forward(batch_data)["logits"]
+                # find prediction probabilities
+                if self.params.use_crf_layer:
+                    msg = f"Prediction probabilities cannot be computed when the param " \
+                          f"use_crf_layer is set to True in {self.__class__.__name__}"
+                    raise NotImplementedError(msg)
+                else:
+                    batch_maxes = torch.max(batch_logits, dim=-1)
+                    batch_predictions = batch_maxes.indices.tolist()
+                    batch_probabilities = batch_maxes.values.tolist()
+                for preds, probs, _split_lengths in zip(
+                    batch_predictions, batch_probabilities, batch_data["split_lengths"]
+                ):
+                    prediction_tuples.append(list(zip(preds, probs))[:len(_split_lengths)])
+        if was_training:
+            self.train()
+        return prediction_tuples
+
+    @abstractmethod
+    def _init_core(self) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def _forward_core(self, batch_data):
+        raise NotImplementedError
+
+
+class EmbedderForTokenClassification(BaseTokenClassification):
+
+    def _init_core(self):
+        self.emb_layer = EmbeddingLayer(
+            self.params._num_tokens,
+            self.params.emb_dim,
+            self.params._padding_idx,
+            self.params.pop("_embedding_weights", None),
+            self.params.update_embeddings,
+            1 - self.params.embedder_output_keep_prob
+        )
+        self.out_dim = self.params.emb_dim
+
+    def _forward_core(self, batch_data):
+        seq_ids = batch_data["seq_ids"]  # [BS, SEQ_LEN]
+
+        token_embs = self.emb_layer(seq_ids)  # [BS, SEQ_LEN, self.out_dim]
+
+        batch_data.update({"token_embs": token_embs})
+        return batch_data
+
+
+class LstmForTokenClassification(BaseTokenClassification):
+    """A LSTM module that operates on a batched sequence of token ids. The tokens could be
+    characters or words or sub-words. This module uses an additional input that determines
+    how the sequence of embeddings obtained after the LSTM layers for each instance in the
+    batch, needs to be split. Once split, the sub-groups of embeddings (each sub-group
+    corresponding to a word or a phrase) can be collapsed to 1D representation per sub-group
+    through pooling operations. Finally, this module outputs a 2D representation for each
+    instance in the batch (i.e. [BS, SEQ_LEN', EMB_DIM]).
+    """
+
+    def _init_core(self):
+        self.emb_layer = EmbeddingLayer(
+            self.params._num_tokens,
+            self.params.emb_dim,
+            self.params._padding_idx,
+            self.params.pop("_embedding_weights", None),
+            self.params.update_embeddings,
+            1 - self.params.embedder_output_keep_prob
+        )
+        self.lstm_layer = LstmLayer(
+            self.params.emb_dim,
+            self.params.lstm_hidden_dim,
+            self.params.lstm_num_layers,
+            1 - self.params.lstm_keep_prob,
+            self.params.lstm_bidirectional
+        )
+        self.out_dim = (
+            self.params.lstm_hidden_dim * 2 if self.params.lstm_bidirectional
+            else self.params.lstm_hidden_dim
+        )
+
+    def _forward_core(self, batch_data):
+        seq_ids = batch_data["seq_ids"]  # [BS, SEQ_LEN]
+
+        flattened_split_lengths = [
+            sum(_split_lengths) +
+            (self.encoder.number_of_terminal_tokens if self.params.add_terminals else 0)
+            for _split_lengths in batch_data["split_lengths"]
+        ]
+        flattened_split_lengths = torch.as_tensor(flattened_split_lengths, dtype=torch.long)  # [BS]
+
+        token_embs = self.emb_layer(seq_ids)  # [BS, SEQ_LEN, EMD_DIM]
+        token_embs = self.lstm_layer(
+            token_embs, flattened_split_lengths)  # [BS, SEQ_LEN, self.out_dim]
+
+        batch_data.update({"token_embs": token_embs})
+
+        return batch_data
+
+
+class CharLstmWithWordLstmForTokenClassification(BaseTokenClassification):
+
+    def _prepare_input_encoder(self, examples, **params):
+        if (params.get("tokenizer_type") and
+            TokenizerType(params.get("tokenizer_type")) !=
+            TokenizerType.WHITESPACE_AND_CHAR_DUAL_TOKENIZER
+        ):
+            msg = f"Param 'tokenizer_type' must be " \
+                  f"{TokenizerType.WHITESPACE_AND_CHAR_DUAL_TOKENIZER.value} for " \
+                  f"{self.__class__.__name__}."
+            raise ValueError(msg)
+
+        params = super()._prepare_input_encoder(examples, **params)
+
+        # update params without which can become ambiguous when loading a model
+        params.update({
+            "_char_num_tokens": len(self.encoder.get_char_vocab()),
+            "_char_padding_idx": self.encoder.get_char_pad_token_idx(),
+            "char_add_terminals": params.get("char_add_terminals"),
+            "char_padding_length": params.get("char_padding_length"),
+            "char_emb_dim": params.get("char_emb_dim"),
+        })
+
+        return params
+
+    def _init_core(self):
+        self.char_proj_dim = self.params.get("char_proj_dim") or self.params.emb_dim
+        self.params.update({"char_proj_dim": self.char_proj_dim})
+
+        self.char_emb_layer = EmbeddingLayer(
+            self.params._char_num_tokens,
+            self.params.char_emb_dim,
+            self.params._char_padding_idx,
+            self.params.pop("_char_embedding_weights", None),
+            self.params.update_embeddings
+        )
+        self.char_lstm_layer = LstmLayer(
+            self.params.char_emb_dim,
+            self.params.char_lstm_hidden_dim,
+            self.params.char_lstm_num_layers,
+            1 - self.params.char_lstm_keep_prob,
+            self.params.char_lstm_bidirectional
+        )
+        self.char_dropout = nn.Dropout(
+            p=1 - self.params.char_lstm_keep_prob
+        )
+        self.char_lstm_output_pooling = PoolingLayer(
+            self.params.char_lstm_output_pooling_type
+        )
+        char_out_dim = (
+            self.params.char_lstm_hidden_dim * 2 if self.params.char_lstm_bidirectional
+            else self.params.char_lstm_hidden_dim
+        )
+        self.char_lstm_output_transform = nn.Linear(char_out_dim, self.params.char_proj_dim)
+        self.emb_layer = EmbeddingLayer(
+            self.params._num_tokens,
+            self.params.emb_dim,
+            self.params._padding_idx,
+            self.params.pop("_embedding_weights", None),
+            self.params.update_embeddings,
+            1 - self.params.embedder_output_keep_prob
+        )
+        self.lstm_layer = LstmLayer(
+            self.params.emb_dim + self.params.char_proj_dim,
+            self.params.lstm_hidden_dim,
+            self.params.lstm_num_layers,
+            1 - self.params.lstm_keep_prob,
+            self.params.lstm_bidirectional
+        )
+        self.out_dim = (
+            self.params.lstm_hidden_dim * 2 if self.params.lstm_bidirectional
+            else self.params.lstm_hidden_dim
+        )
+
+    def _forward_core(self, batch_data):
+        char_seq_ids = batch_data["char_seq_ids"]  # List of [BS, SEQ_LEN]
+        char_seq_lengths = batch_data["char_seq_lengths"]  # List of [BS]
+
+        encs = [self.char_emb_layer(_seq_ids) for _seq_ids in char_seq_ids]
+        encs = [self.char_lstm_layer(enc, seq_len) for enc, seq_len in zip(encs, char_seq_lengths)]
+        encs = [self.char_dropout(enc) for enc in encs]
+        encs = [self.char_lstm_output_pooling(enc, seq_len)
+                for enc, seq_len in zip(encs, char_seq_lengths)]
+        encs = pad_sequence(encs, batch_first=True)  # [BS, SEQ_LEN, char_out_dim]
+        char_encs = self.char_lstm_output_transform(encs)  # [BS, SEQ_LEN, self.char_proj_dim]
+
+        flattened_split_lengths = [
+            sum(_split_lengths) +
+            (self.encoder.number_of_terminal_tokens if self.params.add_terminals else 0)
+            for _split_lengths in batch_data["split_lengths"]
+        ]
+        flattened_split_lengths = torch.as_tensor(flattened_split_lengths, dtype=torch.long)  # [BS]
+
+        seq_ids = batch_data["seq_ids"]  # [BS, SEQ_LEN]
+        word_encs = self.emb_layer(seq_ids)  # [BS, SEQ_LEN, self.emb_dim]
+        char_plus_word_encs = torch.cat((char_encs, word_encs), dim=-1)  # [BS, SEQ_LEN, sum(both)]
+        token_embs = self.lstm_layer(
+            char_plus_word_encs, flattened_split_lengths)  # [BS, SEQ_LEN, self.out_dim]
+
+        batch_data.update({"token_embs": token_embs})
+
+        return batch_data
+
+
+class CharCnnWithWordLstmForTokenClassification(BaseTokenClassification):
+
+    def _prepare_input_encoder(self, examples, **params):
+        if (params.get("tokenizer_type") and
+            TokenizerType(params.get("tokenizer_type")) !=
+            TokenizerType.WHITESPACE_AND_CHAR_DUAL_TOKENIZER
+        ):
+            msg = f"Param 'tokenizer_type' must be " \
+                  f"{TokenizerType.WHITESPACE_AND_CHAR_DUAL_TOKENIZER.value} for " \
+                  f"{self.__class__.__name__}."
+            raise ValueError(msg)
+
+        params = super()._prepare_input_encoder(examples, **params)
+
+        # update params without which can become ambiguous when loading a model
+        params.update({
+            "_char_num_tokens": len(self.encoder.get_char_vocab()),
+            "_char_padding_idx": self.encoder.get_char_pad_token_idx(),
+            "char_add_terminals": params.get("char_add_terminals"),
+            "char_padding_length": params.get("char_padding_length"),
+            "char_emb_dim": params.get("char_emb_dim"),
+        })
+
+        return params
+
+    def _init_core(self):
+        self.char_proj_dim = self.params.get("char_proj_dim") or self.params.emb_dim
+        self.params.update({"char_proj_dim": self.char_proj_dim})
+
+        self.char_emb_layer = EmbeddingLayer(
+            self.params._char_num_tokens,
+            self.params.char_emb_dim,
+            self.params._char_padding_idx,
+            self.params.pop("_char_embedding_weights", None),
+            self.params.update_embeddings
+        )
+        self.char_conv_layer = CnnLayer(
+            self.params.char_emb_dim,
+            self.params.char_window_sizes,
+            self.params.char_number_of_windows
+        )
+        self.char_dropout = nn.Dropout(
+            p=1 - self.params.char_cnn_output_keep_prob
+        )
+        char_out_dim = sum(self.params.char_number_of_windows)
+        self.char_cnn_output_transform = nn.Linear(char_out_dim, self.params.char_proj_dim)
+        self.emb_layer = EmbeddingLayer(
+            self.params._num_tokens,
+            self.params.emb_dim,
+            self.params._padding_idx,
+            self.params.pop("_embedding_weights", None),
+            self.params.update_embeddings,
+            1 - self.params.embedder_output_keep_prob
+        )
+        self.lstm_layer = LstmLayer(
+            self.params.emb_dim + self.params.char_proj_dim,
+            self.params.lstm_hidden_dim,
+            self.params.lstm_num_layers,
+            1 - self.params.lstm_keep_prob,
+            self.params.lstm_bidirectional
+        )
+        self.out_dim = (
+            self.params.lstm_hidden_dim * 2 if self.params.lstm_bidirectional
+            else self.params.lstm_hidden_dim
+        )
+
+    def _forward_core(self, batch_data):
+        char_seq_ids = batch_data["char_seq_ids"]  # List of [BS, SEQ_LEN]
+
+        encs = [self.char_emb_layer(_seq_ids) for _seq_ids in char_seq_ids]
+        encs = [self.char_conv_layer(enc) for enc in encs]
+        encs = [self.char_dropout(enc) for enc in encs]
+        encs = pad_sequence(encs, batch_first=True)  # [BS, SEQ_LEN, sum(self.number_of_windows)]
+        char_encs = self.char_cnn_output_transform(encs)  # [BS, SEQ_LEN, self.char_proj_dim]
+
+        flattened_split_lengths = [
+            sum(_split_lengths) +
+            (self.encoder.number_of_terminal_tokens if self.params.add_terminals else 0)
+            for _split_lengths in batch_data["split_lengths"]
+        ]
+        flattened_split_lengths = torch.as_tensor(flattened_split_lengths, dtype=torch.long)  # [BS]
+
+        seq_ids = batch_data["seq_ids"]  # [BS, SEQ_LEN]
+        word_encs = self.emb_layer(seq_ids)  # [BS, SEQ_LEN, self.emb_dim]
+        char_plus_word_encs = torch.cat((char_encs, word_encs), dim=-1)  # [BS, SEQ_LEN, sum(both)]
+        token_embs = self.lstm_layer(
+            char_plus_word_encs, flattened_split_lengths)  # [BS, SEQ_LEN, self.out_dim]
+
+        batch_data.update({"token_embs": token_embs})
+
+        return batch_data
+
+
+class BertForTokenClassification(BaseTokenClassification):
+
+    def fit(self, examples, labels, **params):
+        # overriding base class' method to set params, and then calling base class' .fit()
+
+        embedder_type = params.get("embedder_type", EmbedderType.BERT.value)
+        if EmbedderType(embedder_type) != EmbedderType.BERT:
+            msg = f"{self.name} can only be used with 'embedder_type': " \
+                  f"'{EmbedderType.BERT.value}'. " \
+                  f"Other values passed through config params are not allowed."
+            raise ValueError(msg)
+
+        safe_values = {
+            "num_warmup_steps": 50,
+            "learning_rate": 2e-5,
+            "optimizer": "AdamW",
+            "max_grad_norm": 1.0
+        }
+
+        for k, v in safe_values.items():
+            v_inputted = params.get(k, v)
+            if v != v_inputted:
+                msg = f"{self.name} can be best used with '{k}' equal to '{v}' but found " \
+                      f"the value '{v_inputted}'. Use the non-default value with caution as it " \
+                      f"may lead to unexpected results and longer training times depending on " \
+                      f"the choice of pretrained model."
+                logger.warning(msg)
+            else:
+                params.update({k: v})
+
+        params.update({
+            "embedder_type": embedder_type
+        })
+
+        super().fit(examples, labels, **params)
+
+    def _create_optimizer(self):
+        params = list(self.named_parameters())
+        no_decay = ["bias", 'LayerNorm.bias', "LayerNorm.weight",
+                    'layer_norm.bias', 'layer_norm.weight']
+        optimizer_grouped_parameters = [
+            {'params': [p for n, p in params if not any(nd in n for nd in no_decay)],
+             'weight_decay': 0.01},
+            {'params': [p for n, p in params if any(nd in n for nd in no_decay)],
+             'weight_decay': 0.0}
+        ]
+        optimizer = getattr(torch.optim, self.params.optimizer)(
+            optimizer_grouped_parameters,
+            lr=self.params.learning_rate,
+            eps=1e-08,
+            weight_decay=0.01
+        )
+        return optimizer
+
+    def _create_optimizer_and_scheduler(self, num_training_steps):
+
+        num_warmup_steps = min(int(0.1 * num_training_steps), self.params.num_warmup_steps)
+        self.params.update({"num_warmup_steps": num_warmup_steps})
+
+        # https://github.com/huggingface/transformers/blob/master/src/transformers/optimization.py
+        # refer `get_linear_schedule_with_warmup` method
+        def lr_lambda(current_step: int):
+            if current_step < num_warmup_steps:
+                return float(current_step) / float(max(1, num_warmup_steps))
+            return max(
+                0.0, float(num_training_steps - current_step) / float(
+                    max(1, num_training_steps - num_warmup_steps))
+            )
+
+        # load a torch optimizer
+        optimizer = self._create_optimizer()
+        # load a lr scheduler
+        scheduler = getattr(torch.optim.lr_scheduler, "LambdaLR")(optimizer, lr_lambda)
+        return optimizer, scheduler
+
+    def _get_dumpable_state_dict(self):
+        if not self.params.update_embeddings and not self.params.save_frozen_bert_weights:
+            state_dict = OrderedDict(
+                {k: v for k, v in self.state_dict().items() if not k.startswith("bert_model")}
+            )
+            return state_dict
+        return self.state_dict()
+
+    def _init_core(self):
+
+        self.bert_model = HuggingfaceTransformersContainer(
+            self.params.pretrained_model_name_or_path,
+            cache_lookup=False
+        ).get_transformer_model()
+        if not self.params.update_embeddings:
+            for param in self.bert_model.parameters():
+                param.requires_grad = False
+
+        self.dropout = nn.Dropout(
+            p=1 - self.params.embedder_output_keep_prob
+        )
+        self.out_dim = self.params.emb_dim
+
+    def _forward_core(self, batch_data):
+
+        # refer to https://huggingface.co/docs/transformers/master/en/main_classes/output
+        # for more details on huggingface's bert outputs
+
+        bert_outputs = self.bert_model(**batch_data["hgf_encodings"], return_dict=True)
+
+        # 'last_hidden_state' refers to the tensor output of the final transformer layer (i.e.
+        # before the logit layer) and hence its dimension  [BS, SEQ_LEN, EMD_DIM]
+        last_hidden_state = bert_outputs.get("last_hidden_state")  # [BS, SEQ_LEN, EMD_DIM]
+        if last_hidden_state is None:
+            msg = f"The choice of pretrained bert model " \
+                  f"({self.params.pretrained_model_name_or_path}) " \
+                  f"has no key 'last_hidden_state' in its output dictionary."
+            raise ValueError(msg)
+
+        last_hidden_state = self.dropout(last_hidden_state)
+        batch_data.update({"token_embs": last_hidden_state})
+        return batch_data
+
+
+def get_token_classifier_cls(classifier_type: str, embedder_type: str = None):
+    try:
+        classifier_type = TokenClassificationType(classifier_type)
+    except ValueError as e:
+        msg = f"Neural Nets' token classification module expects classifier_type to be amongst" \
+              f" {[v.value for v in TokenClassificationType.__members__.values()]}" \
+              f" but found '{classifier_type}'."
+        raise ValueError(msg) from e
+
+    try:
+        embedder_type = EmbedderType(embedder_type)
+    except ValueError as e:
+        msg = f"Neural Nets' token classification module expects embedder_type to be amongst" \
+              f" {[v.value for v in EmbedderType.__members__.values()]} " \
+              f" but found '{embedder_type}'."
+        raise ValueError(msg) from e
+
+    if (
+        embedder_type == EmbedderType.BERT and
+        classifier_type not in [TokenClassificationType.EMBEDDER]
+    ):
+        msg = f"To use the embedder_type '{EmbedderType.BERT.value}', " \
+              f"classifier_type must be '{TokenClassificationType.EMBEDDER.value}'."
+        raise ValueError(msg)
+
+    # disambiguation between glove, bert and non-pretrained embedders
+    def _resolve_and_return_embedder_class(_embedder_type):
+        return {
+            EmbedderType.NONE: EmbedderForTokenClassification,
+            EmbedderType.GLOVE: EmbedderForTokenClassification,
+            EmbedderType.BERT: BertForTokenClassification
+        }[_embedder_type]
+
+    return {
+        TokenClassificationType.EMBEDDER: _resolve_and_return_embedder_class(embedder_type),
+        TokenClassificationType.LSTM: LstmForTokenClassification,
+        TokenClassificationType.CNN_LSTM: CharCnnWithWordLstmForTokenClassification,
+        TokenClassificationType.LSTM_LSTM: CharLstmWithWordLstmForTokenClassification,
+    }[classifier_type]
diff --git a/mindmeld/models/tagger_models.py b/mindmeld/models/tagger_models.py
index 0478e1c6e..6b9d4d9b8 100644
--- a/mindmeld/models/tagger_models.py
+++ b/mindmeld/models/tagger_models.py
@@ -25,7 +25,8 @@
     get_seq_tag_accuracy_scorer,
     ingest_dynamic_gazetteer,
 )
-from .model import ModelConfig, Model, PytorchModel
+from .model import ModelConfig, Model, PytorchModel, AbstractModelFactory
+from .nn_utils import get_token_classifier_cls, TokenClassificationType
 from .taggers.crf import ConditionalRandomFields
 from .taggers.memm import MemmModel
 from ..exceptions import MindMeldError
@@ -227,7 +228,7 @@ def fit(self, examples, labels, params=None):
         if len(set(types)) == 0:
             self._no_entities = True
             logger.info(
-                "There are no labels in this label set, so we don't " "fit the model."
+                "There are no labels in this label set, so we don't fit the model."
             )
             return self
         # Extract labels - label encoders are the same accross all entity recognition models
@@ -426,14 +427,184 @@ def load(cls, path):
 
 
 class PytorchTaggerModel(PytorchModel):
-    ALLOWED_CLASSIFIER_TYPES = ["embedder", "cnn", "lstm"]
-    pass
+    ALLOWED_CLASSIFIER_TYPES = [v.value for v in TokenClassificationType.__members__.values()]
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        self._no_entities = False
+        self.types = None
+
+    def _get_model_constructor(self):
+        """Returns the class of the actual underlying model"""
+        classifier_type = self.config.model_settings["classifier_type"]
+        embedder_type = self.config.params.get("embedder_type") \
+            if self.config.params is not None else None
+
+        return get_token_classifier_cls(
+            classifier_type=classifier_type,
+            embedder_type=embedder_type
+        )
+
+    def evaluate(self, examples, labels):
+        """Evaluates a model against the given examples and labels
+
+        Args:
+            examples: A list of examples to predict
+            labels: A list of expected labels
+
+        Returns:
+            ModelEvaluation: an object containing information about the \
+                evaluation
+        """
+        if self._no_entities:
+            logger.info(
+                "There are no labels in this label set, so we don't "
+                "run model evaluation."
+            )
+            return
+
+        predictions = self.predict(examples)
+
+        evaluations = [
+            EvaluatedExample(e, labels[i], predictions[i], None, self.config.label_type)
+            for i, e in enumerate(examples)
+        ]
+
+        model_eval = EntityModelEvaluation(self.config, evaluations)
+        return model_eval
+
+    def fit(self, examples, labels, params=None):
+
+        types = [entity.entity.type for label in labels for entity in label]
+        self.types = types
+        if len(set(types)) == 0:
+            self._no_entities = True
+            logger.info(
+                "There are no labels in this label set, so we don't fit the model."
+            )
+            return self
+
+        if not examples:
+            return self
+
+        # Encode classes
+        self._label_encoder = get_label_encoder(self.config)
+        y = self._label_encoder.encode(labels, examples=examples)
+        flat_y = sum(y, [])
+        encoded_flat_y = self._class_encoder.fit_transform(flat_y).tolist()
+        encoded_y = []
+        start_idx = 0
+        for seq_length in [len(_y) for _y in y]:
+            encoded_y.append(encoded_flat_y[start_idx: start_idx + seq_length])
+            start_idx += seq_length
+        y = list(encoded_y)
+
+        params = params or self.config.params
+        self._set_query_text_type(params, default="normalized_text")
+        examples_texts = self._get_texts_from_examples(examples)
+        self._validate_training_data(examples_texts, y)
+
+        self._clf = self._get_model_constructor()()  # gets the class name only
+        self._clf.fit(examples_texts, y, **(params if params is not None else {}))
+
+        return self
+
+    def predict(self, examples, dynamic_resource=None):
+        del dynamic_resource
+
+        if self._no_entities:
+            return [()]
+
+        # snippet re-used from ./tagger_model.py/TaggerModel._predict_proba()
+        examples_texts = self._get_texts_from_examples(examples)
+        y = self._clf.predict(examples_texts)
+        flat_y = sum(y, [])
+        decoded_flat_y = self._class_encoder.inverse_transform(flat_y).tolist()
+        decoded_y = []
+        start_idx = 0
+        for seq_length in [len(_y) for _y in y]:
+            decoded_y.append(decoded_flat_y[start_idx: start_idx + seq_length])
+            start_idx += seq_length
+        y = list(decoded_y)
+
+        # Decode the tags to labels
+        labels = [
+            self._label_encoder.decode([_y], examples=[example])[0]
+            for _y, example in zip(y, examples)
+        ]
+        return labels
+
+    def predict_proba(self, examples, dynamic_resource=None):
+        del dynamic_resource
+
+        if self._no_entities:
+            return []
+
+        examples_texts = self._get_texts_from_examples(examples)
+        predicted_tags_probas = self._clf.predict_proba(examples_texts)
+        int_tags, probas = zip(*predicted_tags_probas[0])
+        tags = self._class_encoder.inverse_transform(int_tags).tolist()
+        entity_confidence = []
+        entities = self._label_encoder.decode([tags], examples=[examples[0]])[0]
+        for entity in entities:
+            entity_proba = \
+                probas[entity.normalized_token_span.start: entity.normalized_token_span.end + 1]
+            # We assume that the score of the least likely tag in the sequence as the confidence
+            # score of the entire entity sequence
+            entity_confidence.append(min(entity_proba))
+        predicted_labels_scores = tuple(zip(entities, entity_confidence))
+        return predicted_labels_scores
+
+    def _dump(self, path):
+
+        self._clf.dump(path)
+
+        # dump model metadata
+        metadata = {
+            "label_encoder": self._label_encoder,
+            "class_encoder": self._class_encoder,
+            "query_text_type": self._query_text_type,
+            "model_config": self.config,
+            "no_entities": self._no_entities,
+        }
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        joblib.dump(metadata, path)
+
+    @classmethod
+    def load(cls, path):
+
+        # load model metadata
+        metadata = joblib.load(path)
+
+        model = cls(metadata["model_config"])
+
+        model._label_encoder = metadata["label_encoder"]
+        model._class_encoder = metadata["class_encoder"]
+        model._query_text_type = metadata["query_text_type"]
+        model._no_entities = metadata["no_entities"]
+
+        # underneath tagger load
+        model._clf = model._get_model_constructor().load(path)  # .load() is a classmethod
+
+        return model
+
+    def _validate_training_data(self, examples, labels):
+        super()._validate_training_data(examples, labels)
+
+        for ex, label_tokens in zip(examples, labels):
+            ex_tokens = ex.split(" ")
+            if len(ex_tokens) != len(label_tokens):
+                msg = f"Number of tokens in a sentence ({len(ex_tokens)}) must be same as the " \
+                      f"number of tokens in the corresponding token labels " \
+                      f"({len(label_tokens)}) for sentence '{ex}' with labels '{label_tokens}'"
+                raise AssertionError(msg)
 
 
-class AutoTaggerModel:
+class TaggerModelFactory(AbstractModelFactory):
 
     @staticmethod
-    def get_model_class(config: ModelConfig):
+    def get_model_cls(config: ModelConfig):
 
         CLASSES = [TaggerModel, PytorchTaggerModel]
         classifier_type = config.model_settings["classifier_type"]
diff --git a/mindmeld/models/taggers/embeddings.py b/mindmeld/models/taggers/embeddings.py
index 91b22233d..ed7e6a959 100644
--- a/mindmeld/models/taggers/embeddings.py
+++ b/mindmeld/models/taggers/embeddings.py
@@ -13,20 +13,15 @@
 import logging
 import os
 import pickle
-import zipfile
-from urllib.request import urlretrieve
 
 import numpy as np
-from tqdm import tqdm
-
-from ...exceptions import EmbeddingDownloadError
-from ...path import (
-    EMBEDDINGS_FILE_PATH,
-    EMBEDDINGS_FOLDER_PATH,
+from mindmeld.path import (
     PREVIOUSLY_USED_CHAR_EMBEDDINGS_FILE_PATH,
     PREVIOUSLY_USED_WORD_EMBEDDINGS_FILE_PATH,
 )
 
+from ..containers import GloVeEmbeddingsContainer
+
 logger = logging.getLogger(__name__)
 
 GLOVE_DOWNLOAD_LINK = "http://nlp.stanford.edu/data/glove.6B.zip"
@@ -34,154 +29,6 @@
 ALLOWED_WORD_EMBEDDING_DIMENSIONS = [50, 100, 200, 300]
 
 
-class TqdmUpTo(tqdm):
-    """Provides `update_to(n)` which uses `tqdm.update(delta_n)`."""
-
-    def update_to(self, b=1, bsize=1, tsize=None):
-        """Reports update statistics on the download progress.
-
-        Args:
-            b (int): Number of blocks transferred so far [default: 1].
-            bsize (int): Size of each block (in tqdm units) [default: 1].
-            tsize (int): Total size (in tqdm units). If [default: None] remains unchanged.
-        """
-        if tsize is not None:
-            self.total = tsize
-        self.update(b * bsize - self.n)  # will also set self.n = b * bsize
-
-
-class GloVeEmbeddingsContainer:
-    """This class is responsible for the downloading, extraction and storing of
-    word embeddings based on the GloVe format."""
-
-    def __init__(self, token_dimension=300, token_pretrained_embedding_filepath=None):
-
-        self.token_pretrained_embedding_filepath = token_pretrained_embedding_filepath
-
-        self.token_dimension = token_dimension
-
-        if self.token_dimension not in ALLOWED_WORD_EMBEDDING_DIMENSIONS:
-            logger.info(
-                "Token dimension %s not supported, "
-                "chose from these dimensions: %s. "
-                "Selected 300 by default",
-                token_dimension,
-                str(ALLOWED_WORD_EMBEDDING_DIMENSIONS),
-            )
-            self.token_dimension = 300
-
-        self.word_to_embedding = {}
-        self._extract_embeddings()
-
-    def get_pretrained_word_to_embeddings_dict(self):
-        """Returns the word to embedding dict.
-
-        Returns:
-            (dict): word to embedding mapping.
-        """
-        return self.word_to_embedding
-
-    def _download_embeddings_and_return_zip_handle(self):
-
-        logger.info("Downloading embedding from %s", GLOVE_DOWNLOAD_LINK)
-
-        # Make the folder that will contain the embeddings
-        if not os.path.exists(EMBEDDINGS_FOLDER_PATH):
-            os.makedirs(EMBEDDINGS_FOLDER_PATH)
-
-        with TqdmUpTo(
-            unit="B", unit_scale=True, miniters=1, desc=GLOVE_DOWNLOAD_LINK
-        ) as t:
-
-            try:
-                urlretrieve(
-                    GLOVE_DOWNLOAD_LINK, EMBEDDINGS_FILE_PATH, reporthook=t.update_to
-                )
-
-            except ConnectionError as e:
-                logger.error(
-                    "There was an issue downloading from this "
-                    "link %s with the following error: "
-                    "%s",
-                    GLOVE_DOWNLOAD_LINK,
-                    e,
-                )
-                return
-
-            file_name = EMBEDDING_FILE_PATH_TEMPLATE.format(self.token_dimension)
-            zip_file_object = zipfile.ZipFile(EMBEDDINGS_FILE_PATH, "r")
-
-            if file_name not in zip_file_object.namelist():
-                logger.info(
-                    "Embedding file with %s dimensions " "not found",
-                    self.token_dimension,
-                )
-                return
-
-            return zip_file_object
-
-    def _extract_and_map(self, glove_file):
-        for line in glove_file:
-            values = line.split()
-            word = values[0]
-            coefs = np.asarray(values[1:], dtype="float32")
-            self.word_to_embedding[word] = coefs
-
-    def _extract_embeddings(self):
-        file_location = self.token_pretrained_embedding_filepath
-
-        if file_location and os.path.isfile(file_location):
-            logger.info(
-                "Extracting embeddings from provided " "file location %s.",
-                str(file_location),
-            )
-            with open(file_location, "r") as embedding_file:
-                self._extract_and_map(embedding_file)
-            return
-
-        logger.info("Provided file location %s does not exist.", str(file_location))
-
-        file_name = EMBEDDING_FILE_PATH_TEMPLATE.format(self.token_dimension)
-
-        if os.path.isfile(EMBEDDINGS_FILE_PATH):
-            logger.info(
-                "Extracting embeddings from default folder " "location %s.",
-                EMBEDDINGS_FILE_PATH,
-            )
-
-            try:
-                zip_file_object = zipfile.ZipFile(EMBEDDINGS_FILE_PATH, "r")
-                with zip_file_object.open(file_name) as embedding_file:
-                    self._extract_and_map(embedding_file)
-            except zipfile.BadZipFile:
-                logger.warning(
-                    "%s is corrupt. Deleting the zip file and attempting to"
-                    " download the embedding file again",
-                    EMBEDDINGS_FILE_PATH,
-                )
-                os.remove(EMBEDDINGS_FILE_PATH)
-                self._extract_embeddings()
-            except IOError:
-                logger.error(
-                    "An error occurred when reading %s zip file. The file might"
-                    " be corrupt, so try deleting the file and running the program "
-                    "again",
-                    EMBEDDINGS_FILE_PATH,
-                )
-            return
-
-        logger.info("Default folder location %s does not exist.", EMBEDDINGS_FILE_PATH)
-
-        zip_file_object = self._download_embeddings_and_return_zip_handle()
-
-        if not zip_file_object:
-            raise EmbeddingDownloadError("Failed to download embeddings.")
-
-        with zip_file_object.open(file_name) as embedding_file:
-            self._extract_and_map(embedding_file)
-        return
-
-
 class WordSequenceEmbedding:
     """WordSequenceEmbedding encodes a sequence of words into a sequence of fixed
     dimension real-numbered vectors by mapping each word as a vector.
diff --git a/mindmeld/models/taggers/lstm.py b/mindmeld/models/taggers/lstm.py
index 664cbdf87..7447cf3ed 100644
--- a/mindmeld/models/taggers/lstm.py
+++ b/mindmeld/models/taggers/lstm.py
@@ -998,6 +998,8 @@ def dump(self, path):
 
         joblib.dump(variables_to_dump, os.path.join(path, ".feature_extraction_vars"))
 
+        return path
+
     def unload(self):
         self.graph = None
         self.session = None
diff --git a/mindmeld/models/text_models.py b/mindmeld/models/text_models.py
index 1c4655f9e..18bb970e0 100644
--- a/mindmeld/models/text_models.py
+++ b/mindmeld/models/text_models.py
@@ -38,7 +38,9 @@
     WORD_FREQ_RSC,
     WORD_NGRAM_FREQ_RSC,
 )
-from .model import ModelConfig, Model, PytorchModel
+from .model import ModelConfig, Model, PytorchModel, AbstractModelFactory
+from .nn_utils import get_sequence_classifier_cls, SequenceClassificationType
+from ..resource_loader import ProcessedQueryList as PQL
 
 logger = logging.getLogger(__name__)
 
@@ -491,14 +493,136 @@ def _dump(self, path):
 
 
 class PytorchTextModel(PytorchModel):
-    ALLOWED_CLASSIFIER_TYPES = ["embedder", "cnn", "lstm"]
-    pass
+    ALLOWED_CLASSIFIER_TYPES = [v.value for v in SequenceClassificationType.__members__.values()]
+
+    def _get_model_constructor(self):
+        """Returns the class of the actual underlying model"""
+        classifier_type = self.config.model_settings["classifier_type"]
+        embedder_type = self.config.params.get("embedder_type") \
+            if self.config.params is not None else None
+
+        return get_sequence_classifier_cls(
+            classifier_type=classifier_type,
+            embedder_type=embedder_type
+        )
+
+    def evaluate(self, examples, labels):
+        """Evaluates a model against the given examples and labels
+
+        Args:
+            examples: A list of examples to predict
+            labels: A list of expected labels
+
+        Returns:
+            ModelEvaluation: an object containing information about the \
+                evaluation
+        """
+        predictions = self.predict_proba(examples)
+
+        evaluations = [
+            EvaluatedExample(
+                e, labels[i], predictions[i][0], predictions[i][1], self.config.label_type
+            )
+            for i, e in enumerate(examples)
+        ]
+
+        model_eval = StandardModelEvaluation(self.config, evaluations)
+        return model_eval
+
+    def fit(self, examples, labels, params=None):
+
+        if len(set(labels)) <= 1 or not examples:
+            return self
+
+        if not isinstance(examples, PQL.QueryIterator):
+            # pytorch text models are not implemented for role-classifiers, which pass-in an
+            # instance of ListIterator to this fit() method as opposed to QueryIterator in case of
+            # domain- and intent-classifiers
+            msg = f"{self.__class__.__name__}.fit() only accepts QueryIterator as the first " \
+                  f"argument but found type: {type(examples)}. This might happen if trying to" \
+                  f"create a deep neural net based classifier for role classification which is " \
+                  f"currently not supported."
+            raise NotImplementedError(msg)
+
+        # Encode classes
+        y = self._label_encoder.encode(labels)
+        encoded_y = self._class_encoder.fit_transform(y)
+        y = list(encoded_y)
+
+        params = params or self.config.params
+        self._set_query_text_type(params)
+        examples_texts = self._get_texts_from_examples(examples)
+        self._validate_training_data(examples_texts, y)
+
+        self._clf = self._get_model_constructor()()  # gets the class name and then initializes
+        self._clf.fit(examples_texts, y, **(params if params is not None else {}))
+
+        return self
+
+    def predict(self, examples, dynamic_resource=None):
+        del dynamic_resource
+
+        examples_texts = self._get_texts_from_examples(examples)
+        y = self._clf.predict(examples_texts)
+        predictions = self._class_encoder.inverse_transform(y)
+        return self._label_encoder.decode(predictions)
+
+    def predict_proba(self, examples, dynamic_resource=None):
+        del dynamic_resource
+
+        examples_texts = self._get_texts_from_examples(examples)
+
+        # snippet re-used from ./text_model.py/TextModel._predict_proba()
+        predictions = []
+        for row in self._clf.predict_proba(examples_texts):
+            probabilities = {}
+            top_class = None
+            for class_index, proba in enumerate(row):
+                raw_class = self._class_encoder.inverse_transform([class_index])[0]
+                decoded_class = self._label_encoder.decode([raw_class])[0]
+                probabilities[decoded_class] = proba
+                if proba > probabilities.get(top_class, -1.0):
+                    top_class = decoded_class
+            predictions.append((top_class, probabilities))
+
+        return predictions
+
+    def _dump(self, path):
+
+        self._clf.dump(path)
+
+        # dump model metadata
+        metadata = {
+            "label_encoder": self._label_encoder,
+            "class_encoder": self._class_encoder,
+            "query_text_type": self._query_text_type,
+            "model_config": self.config
+        }
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        joblib.dump(metadata, path)
+
+    @classmethod
+    def load(cls, path):
+
+        # load model metadata
+        metadata = joblib.load(path)
+
+        model = cls(metadata["model_config"])
+
+        model._label_encoder = metadata["label_encoder"]
+        model._class_encoder = metadata["class_encoder"]
+        model._query_text_type = metadata["query_text_type"]
+
+        # underneath tagger load
+        model._clf = model._get_model_constructor().load(path)  # .load() is a classmethod
+
+        return model
 
 
-class AutoTextModel:
+class TextModelFactory(AbstractModelFactory):
 
     @staticmethod
-    def get_model_class(config: ModelConfig):
+    def get_model_cls(config: ModelConfig):
 
         CLASSES = [TextModel, PytorchTextModel]
         classifier_type = config.model_settings["classifier_type"]
diff --git a/mindmeld/resource_loader.py b/mindmeld/resource_loader.py
index 6bc8e8828..5d38bc9bc 100644
--- a/mindmeld/resource_loader.py
+++ b/mindmeld/resource_loader.py
@@ -465,7 +465,8 @@ def _load_cached_models(self):
                     # model; which implies that a model was probably not required (eg. only 1 domain
                     # in case of domain classifier or only 1 intent in an intent classifier or no
                     # entity types for an entity recognizer)
-                    logger.info("Could not find the serialized model")
+                    msg = f"No serialized model available at {classifier_file_path}"
+                    logger.info(msg)
                 self._hash_to_model_path[hash_val] = classifier_file_path
 
     def _gaz_needs_build(self, gaz_name):
diff --git a/pylintrc b/pylintrc
index d64add5d6..4232309ef 100644
--- a/pylintrc
+++ b/pylintrc
@@ -285,7 +285,7 @@ ignored-classes=SQLObject, optparse.Values, thread._local, _thread._local
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
 # expressions are accepted.
-generated-members=REQUEST,acl_users,aq_parent
+generated-members=REQUEST,acl_users,aq_parent,torch.*
 
 # List of decorators that create context managers from functions, such as
 # contextlib.contextmanager.
diff --git a/setup.py b/setup.py
index f27a93289..6504f55b6 100644
--- a/setup.py
+++ b/setup.py
@@ -90,12 +90,9 @@
             'tensorflow~=1.2; python_version < "3.7"',
             'tensorflow>=1.13.1,<2.0; python_version >= "3.7"',
         ],
-        "torch": [
-            'torch~=1.7.0; python_version>="3.6"',
-        ],
-        "bert": [
+        "bert": [  # sentence-transformers
             'torch~=1.7.0; python_version>="3.6"',
-            'transformers~=3.5.1; python_version>="3.6"',
+            'transformers~=4.15.0; python_version>="3.6"',
             'sentence-transformers~=0.3; python_version>="3.6"',
             # elasticsearch-py 7.14 breaks backwards compatibility with servers prior to 7.11
             'elasticsearch>=7.0,<7.14',
@@ -103,9 +100,9 @@
         "examples": [
             'connexion>=2.7.0; python_version>="3.6"',
         ],
-        "augment":[
+        "augment": [
             'torch~=1.7.0; python_version>="3.6"',
-            'transformers~=3.5.1; python_version>="3.6"',
+            'transformers~=4.15.0; python_version>="3.6"',
             'sentencepiece==0.1.91'
         ],
         "language_annotator": [
@@ -121,6 +118,13 @@
         "dvc": [
             'pygit2>=1.5.0,<1.7; python_version < "3.7"',
             "dvc>=1.8.1"
-        ]
+        ],
+        "torch": [
+            'torch~=1.7.0; python_version>="3.6"',
+            'pytorch-crf~=0.7.2'
+        ],
+        "transformers": [  # huggingface-transformers
+            'transformers~=4.15.0; python_version>="3.6"',
+        ],
     },
 )


Test patch:
diff --git a/pytest.ini b/pytest.ini
index bca46a197..ed3ce0729 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -8,6 +8,7 @@ markers =
     torch: tests which require the torch extra
     no_tensorflow: tests which require the tensorflow extra to not be present
     bert: tests which require the bert extra
+    transformers: tests which require the transformers extra
     no_bert: tests which require the bert extra to not be present
     es7: tests which require elasticsearch 7
     conversation: tests which cover a back and forth conversation flow
diff --git a/tests/models/nn_utils/test_layers.py b/tests/models/nn_utils/test_layers.py
new file mode 100644
index 000000000..17a338c1a
--- /dev/null
+++ b/tests/models/nn_utils/test_layers.py
@@ -0,0 +1,296 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Tests for `layers` submodule of nn_utils
+"""
+
+import numpy as np
+import pytest
+
+from mindmeld.models.nn_utils.layers import (
+    EmbeddingLayer,
+    CnnLayer,
+    LstmLayer,
+    PoolingLayer,
+    SplittingAndPoolingLayer
+)
+
+
+@pytest.mark.extras
+@pytest.mark.torch
+def test_embedding_layer():
+    import torch
+    from torch.nn.utils.rnn import pad_sequence
+
+    kwargs = {
+        "num_tokens": 5,
+        "emb_dim": 10,
+        "padding_idx": 2,
+    }
+    seq_ids = [[1, 2, 3], [0, 1, 1, 2, 4, 0]]
+
+    # check forward method of the embedding layer
+    emb_layer = EmbeddingLayer(**kwargs)
+    emb_layer.eval()
+    with torch.no_grad():
+        inputs = pad_sequence(
+            [torch.tensor(seq) for seq in seq_ids],
+            padding_value=kwargs["padding_idx"],
+            batch_first=True)
+        outputs = emb_layer(inputs)
+        assert outputs.shape == (2, 6, 10)
+        assert outputs[0][1][0].item() == 0.0
+        assert sum(outputs[0][-1]).item() == 0.0
+
+    # check if inputted embedding_weights are used for the embedding layer initialization
+    embedding_weights = np.random.random((5, 10))
+    kwargs = {**kwargs, "embedding_weights": embedding_weights}
+    emb_layer = EmbeddingLayer(**kwargs)
+    emb_layer.eval()
+    with torch.no_grad():
+        inputs = pad_sequence(
+            [torch.tensor(seq) for seq in seq_ids],
+            padding_value=kwargs["padding_idx"],
+            batch_first=True)
+        outputs = emb_layer(inputs)
+        assert outputs[0][0][-1] == embedding_weights[1][-1]
+
+
+@pytest.mark.extras
+@pytest.mark.torch
+def test_cnn_layer():
+    import torch
+
+    kwargs = {
+        "emb_dim": 10,
+        "kernel_sizes": [2, 3],
+        "num_kernels": [5, 6],
+    }
+
+    # check forward method of the cnn layer
+    cnn_layer = CnnLayer(**kwargs)
+    cnn_layer.eval()
+    with torch.no_grad():
+        batch_size, max_seq_len = 4, 15
+        inputs = torch.randn((batch_size, max_seq_len, kwargs["emb_dim"]))
+        outputs = cnn_layer(inputs)
+        assert len(outputs.shape) == 2
+        assert outputs.shape == (batch_size, sum(kwargs["num_kernels"]))
+
+
+@pytest.mark.extras
+@pytest.mark.torch
+def test_lstm_layer():
+    import torch
+
+    kwargs = {
+        "emb_dim": 10,
+        "hidden_dim": 5,
+        "num_layers": 2,
+        "lstm_dropout": 0.3,
+        "bidirectional": True,
+    }
+
+    # check forward method of the lstm layer
+    lstm_layer = LstmLayer(**kwargs)
+    lstm_layer.eval()
+    with torch.no_grad():
+        batch_size, max_seq_len = 4, 15
+        input_lengths = torch.randint(2, max_seq_len, (batch_size,))
+        max_seq_len = max(input_lengths)
+        inputs = torch.randn((batch_size, max_seq_len, kwargs["emb_dim"]))
+        outputs = lstm_layer(inputs, input_lengths)
+        assert len(outputs.shape) == 3
+        assert outputs.shape == (batch_size, max_seq_len, 2 * kwargs["hidden_dim"])
+
+        if min(input_lengths) != max(input_lengths):  # not all same lengths
+            batch_dim_pointer1 = np.argmin(input_lengths)
+            batch_dim_pointer2 = np.argmax(input_lengths)
+            assert all(outputs[batch_dim_pointer1][-1] == torch.zeros((2 * kwargs["hidden_dim"],)))
+            assert any(outputs[batch_dim_pointer2][-1] != torch.zeros((2 * kwargs["hidden_dim"],)))
+
+    kwargs.update({
+        "bidirectional": False,
+    })
+
+    lstm_layer = LstmLayer(**kwargs)
+    lstm_layer.eval()
+    with torch.no_grad():
+        batch_size, max_seq_len = 4, 15
+        input_lengths = torch.randint(2, max_seq_len, (batch_size,))
+        max_seq_len = max(input_lengths)
+        inputs = torch.randn((batch_size, max_seq_len, kwargs["emb_dim"]))
+        outputs = lstm_layer(inputs, input_lengths)
+        assert len(outputs.shape) == 3
+        assert outputs.shape == (batch_size, max_seq_len, kwargs["hidden_dim"])
+
+        if min(input_lengths) != max(input_lengths):  # not all same lengths
+            batch_dim_pointer1 = np.argmin(input_lengths)
+            batch_dim_pointer2 = np.argmax(input_lengths)
+            assert all(outputs[batch_dim_pointer1][-1] == torch.zeros((kwargs["hidden_dim"],)))
+            assert any(outputs[batch_dim_pointer2][-1] != torch.zeros((kwargs["hidden_dim"],)))
+
+
+@pytest.mark.extras
+@pytest.mark.torch
+def test_pooling_layer():
+    import torch
+
+    # check forward method of the pooling layer for different pooling types
+
+    kwargs = {
+        "pooling_type": "first",
+    }
+    pooling_layer = PoolingLayer(**kwargs)
+    pooling_layer.eval()
+    with torch.no_grad():
+        batch_size, max_seq_len, emb_dim = 4, 15, 10
+        input_lengths = None
+        inputs = torch.randn((batch_size, max_seq_len, emb_dim))
+        outputs = pooling_layer(inputs, input_lengths)
+        assert len(outputs.shape) == 2
+        assert outputs.shape == (batch_size, emb_dim)
+        assert all(outputs.reshape(-1, ) == inputs[:, 0, :].reshape(-1, ))
+
+    kwargs = {
+        "pooling_type": "last",
+    }
+    pooling_layer = PoolingLayer(**kwargs)
+    pooling_layer.eval()
+    with torch.no_grad():
+        batch_size, max_seq_len, emb_dim = 4, 15, 10
+        input_lengths = torch.randint(2, max_seq_len, (batch_size,))
+        max_seq_len = max(input_lengths)
+        inputs = torch.randn((batch_size, max_seq_len, emb_dim))
+        outputs = pooling_layer(inputs, input_lengths)
+        assert len(outputs.shape) == 2
+        assert outputs.shape == (batch_size, emb_dim)
+        for i, _output in enumerate(outputs):
+            assert all(inputs[i, input_lengths[i] - 1] == _output)
+
+        with pytest.raises(ValueError):
+            _ = pooling_layer(inputs, None)
+
+    kwargs = {
+        "pooling_type": "max",
+    }
+    pooling_layer = PoolingLayer(**kwargs)
+    pooling_layer.eval()
+    with torch.no_grad():
+        batch_size, max_seq_len, emb_dim = 4, 15, 10
+        input_lengths = torch.randint(2, max_seq_len, (batch_size,))
+        max_seq_len = max(input_lengths)
+        inputs = torch.randn((batch_size, max_seq_len, emb_dim))
+        outputs = pooling_layer(inputs, input_lengths)
+        assert len(outputs.shape) == 2
+        assert outputs.shape == (batch_size, emb_dim)
+        for i, _output in enumerate(outputs):
+            expected_output, _ = torch.max(inputs[i, :input_lengths[i]], dim=0)
+            assert all(expected_output == _output)
+
+    kwargs = {
+        "pooling_type": "mean",
+    }
+    pooling_layer = PoolingLayer(**kwargs)
+    pooling_layer.eval()
+    with torch.no_grad():
+        batch_size, max_seq_len, emb_dim = 4, 15, 10
+        input_lengths = torch.randint(2, max_seq_len, (batch_size,))
+        max_seq_len = max(input_lengths)
+        inputs = torch.randn((batch_size, max_seq_len, emb_dim))
+        outputs = pooling_layer(inputs, input_lengths)
+        assert len(outputs.shape) == 2
+        assert outputs.shape == (batch_size, emb_dim)
+        for i, _output in enumerate(outputs):
+            expected_output = torch.mean(inputs[i, :input_lengths[i]], dim=0)
+            assert (expected_output - _output).pow(2).sum(-1).sqrt() < 1e-6
+
+
+@pytest.mark.extras
+@pytest.mark.torch
+def test_splitting_and_pooling_layer():
+    import torch
+
+    # check forward method of the pooling layer for select pooling types; other types already
+    # checked through the pooling layer testing
+
+    kwargs = {
+        "pooling_type": "first",
+        "number_of_terminal_tokens": 2,
+    }
+    splitting_and_pooling_layer = SplittingAndPoolingLayer(**kwargs)
+    splitting_and_pooling_layer.eval()
+    with torch.no_grad():
+        batch_size, emb_dim = 4, 10
+        span_lengths = [
+            torch.randint(1, 4, (n_sub_groups,))
+            for n_sub_groups in torch.randint(1, 4, (batch_size,))
+        ]
+        max_sum_lengths = max([sum(_x) for _x in span_lengths])
+        max_n_lengths = max([len(_x) for _x in span_lengths])
+
+        # inputs w/ terminal token embeddings
+        inputs = torch.randn((batch_size, max_sum_lengths + 2, emb_dim))
+        outputs = splitting_and_pooling_layer(inputs, span_lengths, discard_terminals=True)
+        assert len(outputs.shape) == 3
+        assert outputs.shape == (batch_size, max_n_lengths, emb_dim)
+
+        # inputs w/o terminal token embeddings
+        inputs = torch.randn((batch_size, max_sum_lengths, emb_dim))
+        outputs = splitting_and_pooling_layer(inputs, span_lengths, discard_terminals=False)
+        assert len(outputs.shape) == 3
+        assert outputs.shape == (batch_size, max_n_lengths, emb_dim)
+
+        # check if padding done correctly
+        for i, _span_lengths in enumerate(span_lengths):
+            if len(_span_lengths) < max_n_lengths:
+                assert all(outputs[i][-1] == torch.zeros((emb_dim,)))
+            if len(_span_lengths) == max_n_lengths:
+                assert any(outputs[i][-1] != torch.zeros((emb_dim,)))
+
+        # check if splitting is done correctly
+        for i, _span_lengths in enumerate(span_lengths):
+            select_indices = [0, ] + [_x.item() for _x in np.cumsum(_span_lengths.numpy())[:-1]]
+            for j, select_index in enumerate(select_indices):
+                assert all(inputs[i][select_index] == outputs[i][j])
+
+    kwargs = {
+        "pooling_type": "last",
+        "number_of_terminal_tokens": 2,
+    }
+    splitting_and_pooling_layer = SplittingAndPoolingLayer(**kwargs)
+    splitting_and_pooling_layer.eval()
+    with torch.no_grad():
+        batch_size, emb_dim = 4, 10
+        span_lengths = [
+            torch.randint(1, 4, (n_sub_groups,))
+            for n_sub_groups in torch.randint(1, 4, (batch_size,))
+        ]
+        max_sum_lengths = max([sum(_x) for _x in span_lengths])
+        max_n_lengths = max([len(_x) for _x in span_lengths])
+
+        # inputs w/ terminal token embeddings
+        inputs = torch.randn((batch_size, max_sum_lengths + 2, emb_dim))
+        outputs = splitting_and_pooling_layer(inputs, span_lengths, discard_terminals=True)
+        assert len(outputs.shape) == 3
+        assert outputs.shape == (batch_size, max_n_lengths, emb_dim)
+
+        # inputs w/o terminal token embeddings
+        inputs = torch.randn((batch_size, max_sum_lengths, emb_dim))
+        outputs = splitting_and_pooling_layer(inputs, span_lengths, discard_terminals=False)
+        assert len(outputs.shape) == 3
+        assert outputs.shape == (batch_size, max_n_lengths, emb_dim)
+
+        # check if padding done correctly
+        for i, _span_lengths in enumerate(span_lengths):
+            if len(_span_lengths) < max_n_lengths:
+                assert all(outputs[i][-1] == torch.zeros((emb_dim,)))
+            if len(_span_lengths) == max_n_lengths:
+                assert any(outputs[i][-1] != torch.zeros((emb_dim,)))
+
+        # check if splitting is done correctly
+        for i, _span_lengths in enumerate(span_lengths):
+            select_indices = [_x.item() - 1 for _x in np.cumsum(_span_lengths.numpy())]
+            for j, select_index in enumerate(select_indices):
+                assert all(inputs[i][select_index] == outputs[i][j])
diff --git a/tests/models/nn_utils/test_sequence_classification_models.py b/tests/models/nn_utils/test_sequence_classification_models.py
new file mode 100644
index 000000000..04b6eebe0
--- /dev/null
+++ b/tests/models/nn_utils/test_sequence_classification_models.py
@@ -0,0 +1,597 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Tests for `sequence_classification` submodule of nn_utils
+"""
+
+import os
+import shutil
+
+import pytest
+
+from mindmeld import markup
+from mindmeld.models import QUERY_EXAMPLE_TYPE, CLASS_LABEL_TYPE, ModelFactory, ModelConfig
+from mindmeld.models.nn_utils.helpers import get_num_weights_of_model
+from mindmeld.resource_loader import ResourceLoader, ProcessedQueryList
+
+APP_NAME = "kwik_e_mart"
+APP_PATH = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__))), APP_NAME
+)
+GENERATED_TMP_FOLDER = os.path.join(APP_PATH, ".generated/pytorch_module")
+
+
+@pytest.fixture
+def resource_loader(query_factory):
+    """A resource loader"""
+    return ResourceLoader(APP_PATH, query_factory)
+
+
+@pytest.mark.extras
+@pytest.mark.torch
+class TestSequenceClassification:
+    @classmethod
+    def setup_class(cls):
+        data_dict = {
+            "greet": [
+                "Hello",
+                "Hello!",
+                "hey",
+                "what's up",
+                "greetings",
+                "yo",
+                "hi",
+                "hey, how are you?",
+                "hola",
+                "start",
+            ],
+            "exit": [
+                "bye",
+                "goodbye",
+                "until next time",
+                "see ya later",
+                "ttyl",
+                "talk to you later later",
+                "have a nice day",
+                "finish",
+                "gotta go I'm leaving",
+                "I'm done",
+                "that's all",
+            ],
+        }
+        labeled_data = []
+        for intent in data_dict:
+            for text in data_dict[intent]:
+                labeled_data.append(markup.load_query(text, intent=intent))
+        cls.labeled_data = ProcessedQueryList.from_in_memory_list(labeled_data)
+
+    def test_default_embedder(self, resource_loader):
+        """Tests that a fit succeeds"""
+        config = {
+            "model_type": "text",
+            "example_type": QUERY_EXAMPLE_TYPE,
+            "label_type": CLASS_LABEL_TYPE,
+            "model_settings": {"classifier_type": "embedder"},
+            "params": {"emb_dim": 5},  # default embedder_output_pooling_type is "mean"
+        }
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.intents()
+
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {**config, "params": {**config["params"], "embedder_output_pooling_type": "first"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {**config, "params": {**config["params"], "embedder_output_pooling_type": "last"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+    def test_glove_embedder(self, resource_loader):
+        """Tests that a fit succeeds"""
+        config = {
+            "model_type": "text",
+            "example_type": QUERY_EXAMPLE_TYPE,
+            "label_type": CLASS_LABEL_TYPE,
+            "model_settings": {"classifier_type": "embedder"},
+            "params": {  # default embedder_output_pooling_type is "mean"
+                "embedder_type": "glove", "emb_dim": 5
+            },
+        }
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.intents()
+
+        with pytest.raises(ValueError):
+            model = ModelFactory.create_model_from_config(ModelConfig(**config))
+            model.initialize_resources(resource_loader, examples, labels)
+            model.fit(examples, labels)
+
+        config = {**config, "params": {"embedder_type": "glove"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {**config, "params": {**config["params"], "embedder_output_pooling_type": "max"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+    @pytest.mark.transformers
+    def test_bpe_embedder(self, resource_loader):
+        """Tests that a fit succeeds"""
+        config = {
+            "model_type": "text",
+            "example_type": QUERY_EXAMPLE_TYPE,
+            "label_type": CLASS_LABEL_TYPE,
+            "model_settings": {"classifier_type": "embedder"},
+            "params": {  # default embedder_output_pooling_type is "mean"
+                "emb_dim": 30, "tokenizer_type": "bpe-tokenizer", "add_terminals": True
+            },
+        }
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.intents()
+
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {**config, "params": {**config["params"], "embedder_output_pooling_type": "first"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {**config, "params": {**config["params"], "embedder_output_pooling_type": "last"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {**config, "params": {**config["params"], "embedder_output_pooling_type": "max"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {**config, "params": {
+            **config["params"], "embedder_output_pooling_type": "mean_sqrt"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+    @pytest.mark.transformers
+    def test_wordpiece_embedder(self, resource_loader):
+        """Tests that a fit succeeds"""
+        config = {
+            "model_type": "text",
+            "example_type": QUERY_EXAMPLE_TYPE,
+            "label_type": CLASS_LABEL_TYPE,
+            "model_settings": {"classifier_type": "embedder"},
+            "params": {  # default embedder_output_pooling_type is "mean"
+                "emb_dim": 30, "tokenizer_type": "wordpiece-tokenizer", "add_terminals": True
+            },
+        }
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.intents()
+
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {**config, "params": {**config["params"], "embedder_output_pooling_type": "first"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {**config, "params": {**config["params"], "embedder_output_pooling_type": "last"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+    def test_word_cnn(self, resource_loader):
+        """Tests that a fit succeeds"""
+        config = {
+            "model_type": "text",
+            "example_type": QUERY_EXAMPLE_TYPE,
+            "label_type": CLASS_LABEL_TYPE,
+            "model_settings": {"classifier_type": "cnn"},
+            "params": {"emb_dim": 30},
+        }
+
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.intents()
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+    def test_glove_cnn(self, resource_loader):
+        """Tests that a fit succeeds"""
+        config = {
+            "model_type": "text",
+            "example_type": QUERY_EXAMPLE_TYPE,
+            "label_type": CLASS_LABEL_TYPE,
+            "model_settings": {"classifier_type": "cnn"},
+            "params": {"embedder_type": "glove"},
+        }
+
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.intents()
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+    @pytest.mark.transformers
+    def test_bpe_cnn(self, resource_loader):
+        """Tests that a fit succeeds"""
+        config = {
+            "model_type": "text",
+            "example_type": QUERY_EXAMPLE_TYPE,
+            "label_type": CLASS_LABEL_TYPE,
+            "model_settings": {"classifier_type": "cnn"},
+            "params": {"emb_dim": 30, "tokenizer_type": "bpe-tokenizer", "add_terminals": True},
+        }
+
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.intents()
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+    @pytest.mark.transformers
+    def test_wordpiece_cnn(self, resource_loader):
+        """Tests that a fit succeeds"""
+        config = {
+            "model_type": "text",
+            "example_type": QUERY_EXAMPLE_TYPE,
+            "label_type": CLASS_LABEL_TYPE,
+            "model_settings": {"classifier_type": "cnn"},
+            "params": {
+                "emb_dim": 30, "tokenizer_type": "wordpiece-tokenizer", "add_terminals": True
+            },
+        }
+
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.intents()
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+    def test_char_cnn(self, resource_loader):
+        """Tests that a fit succeeds"""
+        config = {
+            "model_type": "text",
+            "example_type": QUERY_EXAMPLE_TYPE,
+            "label_type": CLASS_LABEL_TYPE,
+            "model_settings": {"classifier_type": "cnn"},
+            "params": {"emb_dim": 30, "tokenizer_type": "char-tokenizer"},
+        }
+
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.intents()
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+    def test_word_lstm(self, resource_loader):
+        """Tests that a fit succeeds"""
+        config = {
+            "model_type": "text",
+            "example_type": QUERY_EXAMPLE_TYPE,
+            "label_type": CLASS_LABEL_TYPE,
+            "model_settings": {"classifier_type": "lstm"},
+            "params": {"emb_dim": 30},
+        }
+
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.intents()
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+    def test_glove_lstm(self, resource_loader):
+        """Tests that a fit succeeds"""
+        config = {
+            "model_type": "text",
+            "example_type": QUERY_EXAMPLE_TYPE,
+            "label_type": CLASS_LABEL_TYPE,
+            "model_settings": {"classifier_type": "lstm"},
+            "params": {"embedder_type": "glove"},
+        }
+
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.intents()
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+    @pytest.mark.transformers
+    def test_bpe_lstm(self, resource_loader):
+        """Tests that a fit succeeds"""
+        config = {
+            "model_type": "text",
+            "example_type": QUERY_EXAMPLE_TYPE,
+            "label_type": CLASS_LABEL_TYPE,
+            "model_settings": {"classifier_type": "lstm"},
+            "params": {  # default lstm_output_pooling_type is "last"
+                "emb_dim": 30, "tokenizer_type": "bpe-tokenizer", "add_terminals": True
+            },
+        }
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.intents()
+
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {**config, "params": {**config["params"], "lstm_output_pooling_type": "first"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {**config, "params": {**config["params"], "lstm_output_pooling_type": "mean"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {**config, "params": {**config["params"], "lstm_output_pooling_type": "max"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {**config, "params": {
+            **config["params"], "lstm_output_pooling_type": "mean_sqrt"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+    @pytest.mark.transformers
+    def test_wordpiece_lstm(self, resource_loader):
+        """Tests that a fit succeeds"""
+        config = {
+            "model_type": "text",
+            "example_type": QUERY_EXAMPLE_TYPE,
+            "label_type": CLASS_LABEL_TYPE,
+            "model_settings": {"classifier_type": "lstm"},
+            "params": {
+                "emb_dim": 30, "tokenizer_type": "wordpiece-tokenizer", "add_terminals": True
+            },
+        }
+
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.intents()
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+    def test_char_lstm(self, resource_loader):
+        """Tests that a fit succeeds"""
+        config = {
+            "model_type": "text",
+            "example_type": QUERY_EXAMPLE_TYPE,
+            "label_type": CLASS_LABEL_TYPE,
+            "model_settings": {"classifier_type": "lstm"},
+            "params": {"emb_dim": 30, "tokenizer_type": "char-tokenizer"},
+        }
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.intents()
+
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {**config, "params": {**config["params"], "add_terminals": "True"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+    @pytest.mark.xfail(strict=False)
+    @pytest.mark.transformers
+    @pytest.mark.bert
+    def test_bert_embedder(self, resource_loader):
+        """Tests that a fit succeeds"""
+        config = {
+            "model_type": "text",
+            "example_type": QUERY_EXAMPLE_TYPE,
+            "label_type": CLASS_LABEL_TYPE,
+            "model_settings": {"classifier_type": "embedder"},
+            "params": {  # default embedder_output_pooling_type for bert is "first"
+                "embedder_type": "bert"
+            },
+        }
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.intents()
+
+        """ test different configurations for bert-base-cased model"""
+
+        config = {**config, "params": {
+            **config["params"],
+            "pretrained_model_name_or_path": "bert-base-cased"
+        }}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {**config, "params": {**config["params"], "embedder_output_pooling_type": "mean"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {**config, "params": {**config["params"], "embedder_output_pooling_type": "last"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {**config, "params": {**config["params"], "embedder_output_pooling_type": "max"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {**config, "params": {
+            **config["params"], "embedder_output_pooling_type": "mean_sqrt"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        """ test for different pretrained transformers"""
+
+        config = {
+            **config,
+            "params": {"pretrained_model_name_or_path": "distilbert-base-uncased", }
+        }
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {
+            **config,
+            "params": {"pretrained_model_name_or_path": "roberta-base"}
+        }
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {
+            **config,
+            "params": {"pretrained_model_name_or_path": "albert-base-v2"}
+        }
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {
+            **config,
+            "params": {"pretrained_model_name_or_path": "sentence-transformers/all-mpnet-base-v2"}
+        }
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+        config = {**config, "params": {**config["params"], "embedder_output_pooling_type": "mean"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+    @pytest.mark.skip(reason="dumping of torch module state dict occupies disk space")
+    @pytest.mark.xfail(strict=False)
+    @pytest.mark.transformers
+    @pytest.mark.bert
+    def test_bert_embedder_frozen_params(self, resource_loader):
+        """Tests that a fit succeeds"""
+        config = {
+            "model_type": "text",
+            "example_type": QUERY_EXAMPLE_TYPE,
+            "label_type": CLASS_LABEL_TYPE,
+            "model_settings": {"classifier_type": "embedder"},
+            "params": {  # default embedder_output_pooling_type for bert is "first"
+                "embedder_type": "bert",
+                "pretrained_model_name_or_path": "distilbert-base-uncased",
+                "embedder_output_pooling_type": "mean",
+                "update_embeddings": False
+            },
+        }
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.intents()
+
+        # fit the model
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+
+        # assert only some weights are trainable
+        clf = model._clf
+        n_requires_grad, n_total = get_num_weights_of_model(clf)
+        assert n_requires_grad < n_total, print(n_requires_grad, n_total)
+
+        # check if dumping and loading partial state dict logs required messages & throws no errors
+        os.makedirs(GENERATED_TMP_FOLDER, exist_ok=True)
+        clf.dump(GENERATED_TMP_FOLDER)
+        new_clf = clf.load(GENERATED_TMP_FOLDER)
+        shutil.rmtree(GENERATED_TMP_FOLDER)
+
+        # do predictions with loaded model
+        model._clf = new_clf
+        assert model.predict([markup.load_query("hi").query])[0] in ["greet", "exit"]
+
+    def test_bert_cnn(self, resource_loader):
+        """Tests that a fit succeeds"""
+        config = {
+            "model_type": "text",
+            "example_type": QUERY_EXAMPLE_TYPE,
+            "label_type": CLASS_LABEL_TYPE,
+            "model_settings": {"classifier_type": "cnn"},
+            "params": {"embedder_type": "bert", "pretrained_model_name_or_path": "bert-base-cased"},
+        }
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.intents()
+
+        # To use a embedder_type 'bert', classifier_type must be 'embedder'.
+        with pytest.raises(ValueError):
+            model = ModelFactory.create_model_from_config(ModelConfig(**config))
+            model.initialize_resources(resource_loader, examples, labels)
+            model.fit(examples, labels)
+
+    def test_bert_lstm(self, resource_loader):
+        """Tests that a fit succeeds"""
+        config = {
+            "model_type": "text",
+            "example_type": QUERY_EXAMPLE_TYPE,
+            "label_type": CLASS_LABEL_TYPE,
+            "model_settings": {"classifier_type": "lstm"},
+            "params": {"embedder_type": "bert", "pretrained_model_name_or_path": "bert-base-cased"},
+        }
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.intents()
+
+        # To use a embedder_type 'bert', classifier_type must be 'embedder'.
+        with pytest.raises(ValueError):
+            model = ModelFactory.create_model_from_config(ModelConfig(**config))
+            model.initialize_resources(resource_loader, examples, labels)
+            model.fit(examples, labels)
diff --git a/tests/models/nn_utils/test_token_classification_models.py b/tests/models/nn_utils/test_token_classification_models.py
new file mode 100644
index 000000000..ae44427bd
--- /dev/null
+++ b/tests/models/nn_utils/test_token_classification_models.py
@@ -0,0 +1,668 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Tests for `token_classification` submodule of nn_utils
+"""
+
+import os
+import shutil
+
+import pytest
+
+from mindmeld import markup
+from mindmeld.models import ENTITY_EXAMPLE_TYPE, ENTITIES_LABEL_TYPE, ModelFactory, ModelConfig
+from mindmeld.models.nn_utils.helpers import get_num_weights_of_model
+from mindmeld.query_factory import QueryFactory
+from mindmeld.resource_loader import ResourceLoader, ProcessedQueryList
+
+APP_NAME = "kwik_e_mart"
+APP_PATH = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__))), APP_NAME
+)
+GENERATED_TMP_FOLDER = os.path.join(APP_PATH, ".generated/pytorch_module")
+QUERY_FACTORY = QueryFactory.create_query_factory(app_path=None, duckling=True)
+
+
+@pytest.fixture
+def resource_loader():
+    """A resource loader"""
+    return ResourceLoader(app_path=None, query_factory=QUERY_FACTORY)
+
+
+def model_predictions_assertions(model):
+    """Conducts assertions on model predictions; common checks across multiple unittests"""
+    predictions = model.predict([
+        markup.load_query("Medium Beers pizza from oz pizza",
+                          query_factory=QUERY_FACTORY).query])[0]
+    assert len(predictions) <= 6
+    for prediction in predictions:
+        if prediction:  # non entities are predicted as NoneType
+            assert prediction.entity.type in {
+                'category', 'cuisine', 'dish', 'option', 'restaurant', 'sys_number'
+            }
+
+
+@pytest.mark.extras
+@pytest.mark.torch
+class TestSequenceClassification:
+    @classmethod
+    def setup_class(cls):
+        examples = [
+            "make that {one|sys_number|num_orders} plate of {vegetable lumpia|dish} and "
+            "{ten|sys_number|num_orders} orders of {pork sasig|dish}",
+            "{lamb curry|dish} with a {side of rice|option} from the {burmese cafe|restaurant}",
+            "i'll get a {normal cheese pizza|dish} with {extra cheese|option} and "
+            "{extra sauce|option}",
+            "What do you have for {dinner|category}?",
+            "i want a {half ramen and omusubi|dish} pork",
+            "{appetizers|category}",
+            "can i get a {regular poki bowl|dish} with {tuna|option} and {yellow tail|option}",
+            "I'd like the {Tabouleh|dish} with {feta cheese|option}, {dolmas|dish} "
+            "and a {prawn kebab plate|dish} from {saffron 685|restaurant}.",
+            "add an order of {tea|dish}",
+            "{Half tofu barbeque pork|dish} and a {coconut water|dish}",
+            "add an order of {tea|dish} from {Sallys|restaurant}",
+            "i feel like having some {Kale Caesar Salad|dish}",
+            "i would like to order from {k-oz restaurant|restaurant}",
+            "Can I get some {south indian|cuisine} food delivered in 30 min?",
+            "{Chorizo con huevos|dish} {without beans|option}",
+            "Can I get a {lamb curry|dish} with a side of {brown rice|option} and {raita|option}?",
+            "can you recommend some {Japan|cuisine} restaurants which are highly reviewed but not "
+            "that expensive",
+            "add the {spicy poke sauce|option}",
+            "{basil eggplant with chicken and prawns|dish}",
+            "{blt|dish} with {egg|option}, make it with {wheat bread|option}",
+            "{blackened catfish|dish} with {no sweet potatoes|category}",
+            "{1|sys_number|num_orders} order of {samosa|dish}, {1|sys_number|num_orders} "
+            "{baingan bartha|dish}, {1|sys_number|num_orders} {chicken achar|dish}, "
+            "{2|sys_number|num_orders} {onion naans|dish} and {1|sys_number|num_orders} "
+            "{kheema naan|dish}",
+            "can i order a {mimosa|dish} {brunch|category}?",
+            "get me something from {Dinosaurs Vietnamese Sandwiches|restaurant}",
+            "what kind of {fish|option} dishes are there",
+            "From {burma cafe|restaurant}, I'll get the {pumpkin curry|dish} with "
+            "{coconut rice|option}",
+            "Can I add get {3|sys_number|num_orders} {guaranas|dish} to that?",
+            "Could you add a {dozen|sys_number|num_orders} {sesame cookies|dish}?",
+            "Give me some of that {pork sisig|dish}",
+            "Give me some food please",
+            "add that",
+            "no expensive food please",
+        ]
+        labeled_data = [markup.load_query(ex, query_factory=QUERY_FACTORY) for ex in examples]
+        cls.labeled_data = ProcessedQueryList.from_in_memory_list(labeled_data)
+
+    def test_default_embedder(self, resource_loader):
+        """Tests that a fit succeeds w/ and w/o crf layer"""
+        config = {
+            "model_type": "tagger",
+            "example_type": ENTITY_EXAMPLE_TYPE,
+            "label_type": ENTITIES_LABEL_TYPE,
+            "model_settings": {"classifier_type": "embedder"},
+            "params": {"emb_dim": 5},
+        }
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.entities()
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "use_crf_layer": False}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+    def test_glove_embedder(self, resource_loader):
+        """Tests that a fit succeeds w/ and w/o crf layer"""
+        config = {
+            "model_type": "tagger",
+            "example_type": ENTITY_EXAMPLE_TYPE,
+            "label_type": ENTITIES_LABEL_TYPE,
+            "model_settings": {"classifier_type": "embedder"},
+            "params": {"embedder_type": "glove", "emb_dim": 5},
+        }
+
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.entities()
+
+        with pytest.raises(ValueError):
+            model = ModelFactory.create_model_from_config(ModelConfig(**config))
+            model.initialize_resources(resource_loader, examples, labels)
+            model.fit(examples, labels)
+
+        config = {**config, "params": {"embedder_type": "glove"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "use_crf_layer": False}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+    @pytest.mark.transformers
+    def test_bpe_embedder(self, resource_loader):
+        """Tests that a fit succeeds w/ and w/o crf layer"""
+        config = {
+            "model_type": "tagger",
+            "example_type": ENTITY_EXAMPLE_TYPE,
+            "label_type": ENTITIES_LABEL_TYPE,
+            "model_settings": {"classifier_type": "embedder"},
+            "params": {  # default token_spans_pooling_type is "first"
+                "emb_dim": 30, "tokenizer_type": "bpe-tokenizer", "add_terminals": True
+            },
+        }
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.entities()
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "token_spans_pooling_type": "mean"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "token_spans_pooling_type": "max"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "token_spans_pooling_type": "mean_sqrt"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "token_spans_pooling_type": "last"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {
+            **config["params"], "use_crf_layer": False, "token_spans_pooling_type": "first"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+    @pytest.mark.transformers
+    def test_wordpiece_embedder(self, resource_loader):
+        """Tests that a fit succeeds w/ and w/o crf layer"""
+        config = {
+            "model_type": "tagger",
+            "example_type": ENTITY_EXAMPLE_TYPE,
+            "label_type": ENTITIES_LABEL_TYPE,
+            "model_settings": {"classifier_type": "embedder"},
+            "params": {
+                "emb_dim": 30, "tokenizer_type": "wordpiece-tokenizer", "add_terminals": True
+            },
+        }
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.entities()
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "use_crf_layer": False}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+    def test_char_embedder(self, resource_loader):
+        """Tests that a fit succeeds w/ and w/o crf layer"""
+        config = {
+            "model_type": "tagger",
+            "example_type": ENTITY_EXAMPLE_TYPE,
+            "label_type": ENTITIES_LABEL_TYPE,
+            "model_settings": {"classifier_type": "embedder"},
+            "params": {  # default token_spans_pooling_type is "first"
+                "emb_dim": 30, "tokenizer_type": "char-tokenizer"},
+        }
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.entities()
+
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "add_terminals": "True"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "token_spans_pooling_type": "mean"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "use_crf_layer": False}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+    def test_word_lstm(self, resource_loader):
+        """Tests that a fit succeeds w/ and w/o crf layer"""
+        config = {
+            "model_type": "tagger",
+            "example_type": ENTITY_EXAMPLE_TYPE,
+            "label_type": ENTITIES_LABEL_TYPE,
+            "model_settings": {"classifier_type": "lstm-pytorch"},
+            "params": {"emb_dim": 30},
+        }
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.entities()
+
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "add_terminals": "True"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "use_crf_layer": False}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+    def test_glove_lstm(self, resource_loader):
+        """Tests that a fit succeeds w/ and w/o crf layer"""
+        config = {
+            "model_type": "tagger",
+            "example_type": ENTITY_EXAMPLE_TYPE,
+            "label_type": ENTITIES_LABEL_TYPE,
+            "model_settings": {"classifier_type": "lstm-pytorch"},
+            "params": {"embedder_type": "glove"},
+        }
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.entities()
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "use_crf_layer": False}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+    @pytest.mark.transformers
+    def test_bpe_lstm(self, resource_loader):
+        """Tests that a fit succeeds w/ and w/o crf layer"""
+        config = {
+            "model_type": "tagger",
+            "example_type": ENTITY_EXAMPLE_TYPE,
+            "label_type": ENTITIES_LABEL_TYPE,
+            "model_settings": {"classifier_type": "lstm-pytorch"},
+            "params": {  # default token_spans_pooling_type is "first"
+                "emb_dim": 30, "tokenizer_type": "bpe-tokenizer", "add_terminals": True
+            },
+        }
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.entities()
+
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "use_crf_layer": False}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "token_spans_pooling_type": "mean"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "token_spans_pooling_type": "max"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "token_spans_pooling_type": "mean_sqrt"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "token_spans_pooling_type": "last"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {
+            **config["params"], "use_crf_layer": False, "token_spans_pooling_type": "first"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+    @pytest.mark.transformers
+    def test_wordpiece_lstm(self, resource_loader):
+        """Tests that a fit succeeds w/ and w/o crf layer"""
+        config = {
+            "model_type": "tagger",
+            "example_type": ENTITY_EXAMPLE_TYPE,
+            "label_type": ENTITIES_LABEL_TYPE,
+            "model_settings": {"classifier_type": "lstm-pytorch"},
+            "params": {
+                "emb_dim": 30, "tokenizer_type": "wordpiece-tokenizer", "add_terminals": True
+            },
+        }
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.entities()
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "use_crf_layer": False}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+    def test_char_lstm(self, resource_loader):
+        """Tests that a fit succeeds w/ and w/o crf layer"""
+        config = {
+            "model_type": "tagger",
+            "example_type": ENTITY_EXAMPLE_TYPE,
+            "label_type": ENTITIES_LABEL_TYPE,
+            "model_settings": {"classifier_type": "lstm-pytorch"},
+            "params": {  # default token_spans_pooling_type is "first"
+                "embedder_type": "glove", "emb_dim": 30, "tokenizer_type": "char-tokenizer"},
+        }
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.entities()
+
+        with pytest.raises(ValueError):
+            model = ModelFactory.create_model_from_config(ModelConfig(**config))
+            model.initialize_resources(resource_loader, examples, labels)
+            model.fit(examples, labels)
+
+        config = {**config, "params": {
+            "embedder_type": None, "emb_dim": 30, "tokenizer_type": "char-tokenizer"}
+        }
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "add_terminals": "True"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "token_spans_pooling_type": "mean"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "use_crf_layer": False}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+    def test_char_lstm_word_lstm(self, resource_loader):
+        """Tests that a fit succeeds w/ and w/o crf layer"""
+        config = {
+            "model_type": "tagger",
+            "example_type": ENTITY_EXAMPLE_TYPE,
+            "label_type": ENTITIES_LABEL_TYPE,
+            "model_settings": {"classifier_type": "lstm-lstm"},
+            "params": {"emb_dim": 5},
+        }
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.entities()
+
+        incorrect_config = {**config, "params": {**config["params"], "add_terminals": True}}
+        with pytest.raises(ValueError):
+            model = ModelFactory.create_model_from_config(ModelConfig(**incorrect_config))
+            model.initialize_resources(resource_loader, examples, labels)
+            model.fit(examples, labels)
+
+        incorrect_config = {
+            **config, "params": {**config["params"], "tokenizer_type": "char-tokenizer"}}
+        with pytest.raises(ValueError):
+            model = ModelFactory.create_model_from_config(ModelConfig(**incorrect_config))
+            model.initialize_resources(resource_loader, examples, labels)
+            model.fit(examples, labels)
+
+        incorrect_config = {
+            **config, "params": {
+                **config["params"], "embedder_type": "bert",
+                "pretrained_model_name_or_path": "bert-base-cased"}
+        }
+        with pytest.raises(ValueError):
+            model = ModelFactory.create_model_from_config(ModelConfig(**incorrect_config))
+            model.initialize_resources(resource_loader, examples, labels)
+            model.fit(examples, labels)
+
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "use_crf_layer": False}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "char_lstm_output_pooling_type": "mean"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        glove_config = {**config, "params": {"embedder_type": "glove"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**glove_config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+    def test_char_cnn_word_lstm(self, resource_loader):
+        """Tests that a fit succeeds w/ and w/o crf layer"""
+        config = {
+            "model_type": "tagger",
+            "example_type": ENTITY_EXAMPLE_TYPE,
+            "label_type": ENTITIES_LABEL_TYPE,
+            "model_settings": {"classifier_type": "cnn-lstm"},
+            "params": {"emb_dim": 5},
+        }
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.entities()
+
+        incorrect_config = {**config, "params": {**config["params"], "add_terminals": True}}
+        with pytest.raises(ValueError):
+            model = ModelFactory.create_model_from_config(ModelConfig(**incorrect_config))
+            model.initialize_resources(resource_loader, examples, labels)
+            model.fit(examples, labels)
+
+        incorrect_config = {
+            **config, "params": {**config["params"], "tokenizer_type": "char-tokenizer"}}
+        with pytest.raises(ValueError):
+            model = ModelFactory.create_model_from_config(ModelConfig(**incorrect_config))
+            model.initialize_resources(resource_loader, examples, labels)
+            model.fit(examples, labels)
+
+        incorrect_config = {
+            **config, "params": {
+                **config["params"], "embedder_type": "bert",
+                "pretrained_model_name_or_path": "bert-base-cased"}
+        }
+        with pytest.raises(ValueError):
+            model = ModelFactory.create_model_from_config(ModelConfig(**incorrect_config))
+            model.initialize_resources(resource_loader, examples, labels)
+            model.fit(examples, labels)
+
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {**config, "params": {**config["params"], "use_crf_layer": False}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        glove_config = {**config, "params": {"embedder_type": "glove"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**glove_config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+    @pytest.mark.xfail(strict=False)
+    @pytest.mark.transformers
+    def test_bert_embedder(self, resource_loader):
+        """Tests that a fit succeeds w/ and w/o crf layer"""
+        config = {
+            "model_type": "tagger",
+            "example_type": ENTITY_EXAMPLE_TYPE,
+            "label_type": ENTITIES_LABEL_TYPE,
+            "model_settings": {"classifier_type": "embedder"},
+            "params": {"embedder_type": "bert"},
+        }
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.entities()
+
+        config = {
+            **config,
+            "params": {
+                "embedder_type": "bert",
+                "pretrained_model_name_or_path": "bert-base-cased",
+                "add_terminals": True
+            }
+        }
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        new_config = {**config, "params": {**config["params"], "token_spans_pooling_type": "mean"}}
+        model = ModelFactory.create_model_from_config(ModelConfig(**new_config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        """ test for different pretrained transformers"""
+
+        config = {
+            **config,
+            "params": {
+                "embedder_type": "bert",
+                "pretrained_model_name_or_path": "distilbert-base-uncased",
+            }
+        }
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {
+            **config,
+            "params": {
+                "embedder_type": "bert",
+                "pretrained_model_name_or_path": "albert-base-v2",
+            }
+        }
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {
+            **config,
+            "params": {
+                "embedder_type": "bert",
+                "pretrained_model_name_or_path": "sentence-transformers/all-mpnet-base-v2",
+            }
+        }
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+        model_predictions_assertions(model)
+
+        config = {
+            **config,
+            "params": {
+                "embedder_type": "bert",
+                "pretrained_model_name_or_path": "roberta-base",
+            }
+        }
+        with pytest.raises(ValueError):
+            model = ModelFactory.create_model_from_config(ModelConfig(**config))
+            model.initialize_resources(resource_loader, examples, labels)
+            model.fit(examples, labels)
+            model_predictions_assertions(model)
+
+    @pytest.mark.skip(reason="dumping of torch module state dict occupies disk space")
+    @pytest.mark.xfail(strict=False)
+    @pytest.mark.transformers
+    @pytest.mark.bert
+    def test_bert_embedder_frozen_params(self, resource_loader):
+        """Tests that a fit succeeds"""
+        config = {
+            "model_type": "tagger",
+            "example_type": ENTITY_EXAMPLE_TYPE,
+            "label_type": ENTITIES_LABEL_TYPE,
+            "model_settings": {"classifier_type": "embedder"},
+            "params": {  # default embedder_output_pooling_type for bert is "first"
+                "embedder_type": "bert",
+                "pretrained_model_name_or_path": "distilbert-base-uncased",
+                "update_embeddings": False
+            },
+        }
+        examples = self.labeled_data.queries()
+        labels = self.labeled_data.entities()
+
+        # fit the model
+        model = ModelFactory.create_model_from_config(ModelConfig(**config))
+        model.initialize_resources(resource_loader, examples, labels)
+        model.fit(examples, labels)
+
+        # assert only some weights are trainable
+        clf = model._clf
+        n_requires_grad, n_total = get_num_weights_of_model(clf)
+        assert n_requires_grad < n_total, print(n_requires_grad, n_total)
+
+        # check if dumping and loading partial state dict logs required messages & throws no errors
+        os.makedirs(GENERATED_TMP_FOLDER, exist_ok=True)
+        clf.dump(GENERATED_TMP_FOLDER)
+        new_clf = clf.load(GENERATED_TMP_FOLDER)
+        shutil.rmtree(GENERATED_TMP_FOLDER)
+
+        # do predictions with loaded model
+        model._clf = new_clf
+        model_predictions_assertions(model)
diff --git a/tests/models/test_model_factory.py b/tests/models/test_model_factory.py
new file mode 100644
index 000000000..343505b02
--- /dev/null
+++ b/tests/models/test_model_factory.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import pytest
+
+from mindmeld.models import (
+    ModelFactory,
+    create_model,
+    CLASS_LABEL_TYPE,
+    ENTITIES_LABEL_TYPE,
+    QUERY_EXAMPLE_TYPE,
+    ENTITY_EXAMPLE_TYPE,
+    ModelConfig
+)
+from mindmeld.models.tagger_models import PytorchTaggerModel
+from mindmeld.models.text_models import TextModel, PytorchTextModel
+
+
+def test_create_model_from_config_dict():
+    config = {
+        "model_type": "text",
+        "example_type": QUERY_EXAMPLE_TYPE,
+        "label_type": CLASS_LABEL_TYPE,
+        "model_settings": {"classifier_type": "logreg"},
+        "params": {"fit_intercept": True, "C": 100},
+        "features": {
+            "bag-of-words": {"lengths": [1]},
+            "freq": {"bins": 5},
+            "length": {},
+        },
+    }
+    model = ModelFactory.create_model_from_config(model_config=config)
+    assert isinstance(model, TextModel)
+
+
+@pytest.mark.extras
+@pytest.mark.torch
+def test_create_model_from_config_object():
+    config = {
+        "model_type": "text",
+        "example_type": QUERY_EXAMPLE_TYPE,
+        "label_type": CLASS_LABEL_TYPE,
+        "model_settings": {"classifier_type": "lstm"},
+    }
+    model = ModelFactory.create_model_from_config(model_config=ModelConfig(**config))
+    assert isinstance(model, PytorchTextModel)
+
+
+@pytest.mark.extras
+@pytest.mark.torch
+def test_create_model_from_helpers():
+    config = {
+        "model_type": "tagger",
+        "example_type": ENTITY_EXAMPLE_TYPE,
+        "label_type": ENTITIES_LABEL_TYPE,
+        "model_settings": {"classifier_type": "lstm-pytorch"},
+        "params": {},
+    }
+    model = create_model(config=ModelConfig(**config))
+    assert isinstance(model, PytorchTaggerModel)
+
+
+@pytest.mark.extras
+@pytest.mark.torch
+def test_create_model_from_helpers_without_input_type():
+    config = {
+        "model_type": "tagger",
+        "model_settings": {"classifier_type": "lstm-pytorch"},
+        "params": {},
+    }
+    with pytest.raises(TypeError):
+        model = create_model(config=ModelConfig(**config))
+        del model
+
+
+def test_create_model_from_incomplete_config_without_model_settings():
+    incomplete_config = {
+        "model_type": "text",
+        "example_type": QUERY_EXAMPLE_TYPE,
+        "label_type": CLASS_LABEL_TYPE,
+    }
+    with pytest.raises(TypeError):
+        model = ModelFactory.create_model_from_config(model_config=incomplete_config)
+        del model
+
+
+@pytest.mark.extras
+@pytest.mark.torch
+def test_create_model_from_incomplete_config_without_params():
+    incomplete_config = {
+        "model_type": "tagger",
+        "example_type": QUERY_EXAMPLE_TYPE,
+        "label_type": CLASS_LABEL_TYPE,
+        "model_settings": {"classifier_type": "lstm"},
+    }
+    with pytest.raises(ValueError):
+        model = ModelFactory.create_model_from_config(model_config=incomplete_config)
+        del model