|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
""" BERT model configuration """ |
|
|
|
|
|
import logging |
|
|
|
from .configuration_utils import PretrainedConfig |
|
|
|
|
|
logger = logging.getLogger(__name__) |
|
|
|
BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { |
|
"bert-base-uncased": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-config.json", |
|
"bert-large-uncased": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-config.json", |
|
"bert-base-cased": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-config.json", |
|
"bert-large-cased": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-config.json", |
|
"bert-base-multilingual-uncased": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-uncased-config.json", |
|
"bert-base-multilingual-cased": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-config.json", |
|
"bert-base-chinese": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-chinese-config.json", |
|
"bert-base-german-cased": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-cased-config.json", |
|
"bert-large-uncased-whole-word-masking": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-whole-word-masking-config.json", |
|
"bert-large-cased-whole-word-masking": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-whole-word-masking-config.json", |
|
"bert-large-uncased-whole-word-masking-finetuned-squad": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-whole-word-masking-finetuned-squad-config.json", |
|
"bert-large-cased-whole-word-masking-finetuned-squad": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-whole-word-masking-finetuned-squad-config.json", |
|
"bert-base-cased-finetuned-mrpc": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-config.json", |
|
"bert-base-german-dbmdz-cased": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-cased-config.json", |
|
"bert-base-german-dbmdz-uncased": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-uncased-config.json", |
|
"cl-tohoku/bert-base-japanese": "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese/config.json", |
|
"cl-tohoku/bert-base-japanese-whole-word-masking": "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-whole-word-masking/config.json", |
|
"cl-tohoku/bert-base-japanese-char": "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char/config.json", |
|
"cl-tohoku/bert-base-japanese-char-whole-word-masking": "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-whole-word-masking/config.json", |
|
"TurkuNLP/bert-base-finnish-cased-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1/config.json", |
|
"TurkuNLP/bert-base-finnish-uncased-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-uncased-v1/config.json", |
|
"wietsedv/bert-base-dutch-cased": "https://s3.amazonaws.com/models.huggingface.co/bert/wietsedv/bert-base-dutch-cased/config.json", |
|
|
|
} |
|
|
|
|
|
class BertConfig(PretrainedConfig): |
|
r""" |
|
This is the configuration class to store the configuration of a :class:`~transformers.BertModel`. |
|
It is used to instantiate an BERT model according to the specified arguments, defining the model |
|
architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of |
|
the BERT `bert-base-uncased <https://huggingface.co/bert-base-uncased>`__ architecture. |
|
|
|
Configuration objects inherit from :class:`~transformers.PretrainedConfig` and can be used |
|
to control the model outputs. Read the documentation from :class:`~transformers.PretrainedConfig` |
|
for more information. |
|
|
|
|
|
Args: |
|
vocab_size (:obj:`int`, optional, defaults to 30522): |
|
Vocabulary size of the BERT model. Defines the different tokens that |
|
can be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.BertModel`. |
|
hidden_size (:obj:`int`, optional, defaults to 768): |
|
Dimensionality of the encoder layers and the pooler layer. |
|
num_hidden_layers (:obj:`int`, optional, defaults to 12): |
|
Number of hidden layers in the Transformer encoder. |
|
num_attention_heads (:obj:`int`, optional, defaults to 12): |
|
Number of attention heads for each attention layer in the Transformer encoder. |
|
intermediate_size (:obj:`int`, optional, defaults to 3072): |
|
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. |
|
hidden_act (:obj:`str` or :obj:`function`, optional, defaults to "gelu"): |
|
The non-linear activation function (function or string) in the encoder and pooler. |
|
If string, "gelu", "relu", "swish" and "gelu_new" are supported. |
|
hidden_dropout_prob (:obj:`float`, optional, defaults to 0.1): |
|
The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler. |
|
attention_probs_dropout_prob (:obj:`float`, optional, defaults to 0.1): |
|
The dropout ratio for the attention probabilities. |
|
max_position_embeddings (:obj:`int`, optional, defaults to 512): |
|
The maximum sequence length that this model might ever be used with. |
|
Typically set this to something large just in case (e.g., 512 or 1024 or 2048). |
|
type_vocab_size (:obj:`int`, optional, defaults to 2): |
|
The vocabulary size of the `token_type_ids` passed into :class:`~transformers.BertModel`. |
|
initializer_range (:obj:`float`, optional, defaults to 0.02): |
|
The standard deviation of the truncated_normal_initializer for initializing all weight matrices. |
|
layer_norm_eps (:obj:`float`, optional, defaults to 1e-12): |
|
The epsilon used by the layer normalization layers. |
|
gradient_checkpointing (:obj:`bool`, optional, defaults to False): |
|
If True, use gradient checkpointing to save memory at the expense of slower backward pass. |
|
|
|
Example:: |
|
|
|
>>> from transformers import BertModel, BertConfig |
|
|
|
>>> # Initializing a BERT bert-base-uncased style configuration |
|
>>> configuration = BertConfig() |
|
|
|
>>> # Initializing a model from the bert-base-uncased style configuration |
|
>>> model = BertModel(configuration) |
|
|
|
>>> # Accessing the model configuration |
|
>>> configuration = model.config |
|
""" |
|
model_type = "bert" |
|
|
|
def __init__( |
|
self, |
|
vocab_size=30522, |
|
hidden_size=768, |
|
num_hidden_layers=12, |
|
num_attention_heads=12, |
|
intermediate_size=3072, |
|
hidden_act="gelu", |
|
hidden_dropout_prob=0.1, |
|
attention_probs_dropout_prob=0.1, |
|
max_position_embeddings=512, |
|
type_vocab_size=2, |
|
initializer_range=0.02, |
|
layer_norm_eps=1e-12, |
|
pad_token_id=0, |
|
gradient_checkpointing=False, |
|
**kwargs |
|
): |
|
super().__init__(pad_token_id=pad_token_id, **kwargs) |
|
|
|
self.vocab_size = vocab_size |
|
self.hidden_size = hidden_size |
|
self.num_hidden_layers = num_hidden_layers |
|
self.num_attention_heads = num_attention_heads |
|
self.hidden_act = hidden_act |
|
self.intermediate_size = intermediate_size |
|
self.hidden_dropout_prob = hidden_dropout_prob |
|
self.attention_probs_dropout_prob = attention_probs_dropout_prob |
|
self.max_position_embeddings = max_position_embeddings |
|
self.type_vocab_size = type_vocab_size |
|
self.initializer_range = initializer_range |
|
self.layer_norm_eps = layer_norm_eps |
|
self.gradient_checkpointing = gradient_checkpointing |
|
|
