# Buggy code
def load(cls, pretrained_model_name_or_path, language=None, **kwargs):
    """
    Load a pretrained model by supplying

    * the name of a remote model on s3 ("distilbert-base-german-cased" ...)
    * OR a local path of a model trained via transformers ("some_dir/huggingface_model")
    * OR a local path of a model trained via FARM ("some_dir/farm_model")

    :param pretrained_model_name_or_path: The path of the saved pretrained model or its name.
    :type pretrained_model_name_or_path: str

    """

    distilbert = cls()
    if "farm_lm_name" in kwargs:
        distilbert.name = kwargs["farm_lm_name"]
    else:
        distilbert.name = pretrained_model_name_or_path
    # We need to differentiate between loading model using FARM format and Pytorch-Transformers format
    farm_lm_config = Path(pretrained_model_name_or_path) / "language_model_config.json"
    if os.path.exists(farm_lm_config):
        # FARM style
        config = AlbertConfig.from_pretrained(farm_lm_config)
        farm_lm_model = Path(pretrained_model_name_or_path) / "language_model.bin"
        distilbert.model = DistilBertModel.from_pretrained(
            farm_lm_model, config=config, **kwargs
        )
        distilbert.language = distilbert.model.config.language
    else:
        # Pytorch-transformer Style
        distilbert.model = DistilBertModel.from_pretrained(
            str(pretrained_model_name_or_path), **kwargs
        )
        distilbert.language = cls._get_or_infer_language_from_name(
            language, pretrained_model_name_or_path
        )
    config = distilbert.model.config

    # DistilBERT does not provide a pooled_output by default. Therefore, we need to initialize an extra pooler.
    # The pooler takes the first hidden representation & feeds it to a dense layer of (hidden_dim x hidden_dim).
    # We don't want a dropout in the end of the pooler, since we do that already in the adaptive model before we
    # feed everything to the prediction head
    config.summary_last_dropout = 0
    config.summary_type = "first"
    config.summary_activation = "tanh"
    distilbert.pooler = SequenceSummary(config)
    distilbert.pooler.apply(distilbert.model._init_weights)
    return distilbert


# Traceback (RuntimeError)
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-83-b2a730b6ac24> in <module>
----> 1 convert_to_transformers()

<ipython-input-82-8ab35f02f804> in convert_to_transformers()
12
13     # convert to transformers
---> 14     transformer_model = model.convert_to_transformers()
15
16     # save it (note: transformers use str instead of Path objects)

555             setattr(transformers_model, transformers_model.base_model_prefix, self.language_model.model)
556             transformers_model.classifier.load_state_dict(
--> 557                 self.prediction_heads[0].feed_forward.feed_forward[0].state_dict())
558         elif self.prediction_heads[0].model_type == "token_classification":
559             # add more info to config

1043         if len(error_msgs) > 0:
1044             raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
-> 1045                                self.__class__.__name__, "\n\t".join(error_msgs)))
1046         return _IncompatibleKeys(missing_keys, unexpected_keys)
1047

RuntimeError: Error(s) in loading state_dict for Linear:
size mismatch for weight: copying a param with shape torch.Size([2, 768]) from checkpoint, the shape in current model is torch.Size([2, 4096]).

# Fixed code
def load(cls, pretrained_model_name_or_path, language=None, **kwargs):
    """
    Load a pretrained model by supplying

    * the name of a remote model on s3 ("distilbert-base-german-cased" ...)
    * OR a local path of a model trained via transformers ("some_dir/huggingface_model")
    * OR a local path of a model trained via FARM ("some_dir/farm_model")

    :param pretrained_model_name_or_path: The path of the saved pretrained model or its name.
    :type pretrained_model_name_or_path: str

    """

    distilbert = cls()
    if "farm_lm_name" in kwargs:
        distilbert.name = kwargs["farm_lm_name"]
    else:
        distilbert.name = pretrained_model_name_or_path
    # We need to differentiate between loading model using FARM format and Pytorch-Transformers format
    farm_lm_config = Path(pretrained_model_name_or_path) / "language_model_config.json"
    if os.path.exists(farm_lm_config):
        # FARM style
        config = DistilBertConfig.from_pretrained(farm_lm_config)
        farm_lm_model = Path(pretrained_model_name_or_path) / "language_model.bin"
        distilbert.model = DistilBertModel.from_pretrained(
            farm_lm_model, config=config, **kwargs
        )
        distilbert.language = distilbert.model.config.language
    else:
        # Pytorch-transformer Style
        distilbert.model = DistilBertModel.from_pretrained(
            str(pretrained_model_name_or_path), **kwargs
        )
        distilbert.language = cls._get_or_infer_language_from_name(
            language, pretrained_model_name_or_path
        )
    config = distilbert.model.config

    # DistilBERT does not provide a pooled_output by default. Therefore, we need to initialize an extra pooler.
    # The pooler takes the first hidden representation & feeds it to a dense layer of (hidden_dim x hidden_dim).
    # We don't want a dropout in the end of the pooler, since we do that already in the adaptive model before we
    # feed everything to the prediction head
    config.summary_last_dropout = 0
    config.summary_type = "first"
    config.summary_activation = "tanh"
    distilbert.pooler = SequenceSummary(config)
    distilbert.pooler.apply(distilbert.model._init_weights)
    return distilbert