Mixtral_AI_TokenClassification / configuration_mistral_advanced.py

Upload 2 files

cc45cd2 verified 2 months ago

No virus

20.4 kB

	from typing import TYPE_CHECKING, Any, Mapping, Optional, OrderedDict
	from packaging import version
	from transformers.auto.configuration_auto import AutoConfig
	from transformers.configuration_utils import PretrainedConfig
	from transformers.utils import logging



	if TYPE_CHECKING:
	from ... import PreTrainedTokenizerBase, TensorType

	logger = logging.get_logger(__name__)

	""" Mistral model configuration"""



	MISTRAL_PRETRAINED_CONFIG_ARCHIVE_MAP = {
	"mistralai/Mistral-7B-v0.1": "https://huggingface.co/mistralai/Mistral-7B-v0.1/resolve/main/config.json",
	"mistralai/Mistral-7B-Instruct-v0.1": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1/resolve/main/config.json",
	}

	class EncoderDecoderConfig(PretrainedConfig):
	is_composition = True

	def __init__(self, **kwargs):
	super().__init__(**kwargs)
	if "encoder" not in kwargs or "decoder" not in kwargs:
	raise ValueError(
	f"A configuraton of type {self.model_type} cannot be instantiated because "
	f"not both `encoder` and `decoder` sub-configurations are passed, but only {kwargs}"
	)

	encoder_config = kwargs.pop("encoder")
	encoder_model_type = encoder_config.pop("model_type")
	decoder_config = kwargs.pop("decoder")
	decoder_model_type = decoder_config.pop("model_type")

	self.encoder = AutoConfig.for_model(encoder_model_type, **encoder_config)
	self.decoder = AutoConfig.for_model(decoder_model_type, **decoder_config)
	self.is_encoder_decoder = True
	@classmethod
	def from_encoder_decoder_configs(
	cls, encoder_config: PretrainedConfig, decoder_config: PretrainedConfig, **kwargs
	) -> PretrainedConfig:
	r"""
	Instantiate a [`SpeechEncoderDecoderConfig`] (or a derived class) from a pre-trained encoder model
	configuration and decoder model configuration.

	Returns:
	[`SpeechEncoderDecoderConfig`]: An instance of a configuration object
	"""
	logger.info("Setting `config.is_decoder=True` and `config.add_cross_attention=True` for decoder_config")
	decoder_config.is_decoder = True
	decoder_config.add_cross_attention = True

	return cls(encoder=encoder_config.to_dict(), decoder=decoder_config.to_dict(), **kwargs)

	class VisionEncoderDecoderConfig(PretrainedConfig):
	r"""
	[`VisionEncoderDecoderConfig`] is the configuration class to store the configuration of a
	[`VisionEncoderDecoderModel`]. It is used to instantiate a Vision-Encoder-Text-Decoder model according to the
	specified arguments, defining the encoder and decoder configs.

	Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
	documentation from [`PretrainedConfig`] for more information.

	Args:
	kwargs (optional):
	Dictionary of keyword arguments. Notably:

	- encoder ([`PretrainedConfig`], optional) -- An instance of a configuration object that defines
	the encoder config.
	- decoder ([`PretrainedConfig`], optional) -- An instance of a configuration object that defines
	the decoder config.

	Examples:

	```python
	>>> from transformers import BertConfig, ViTConfig, VisionEncoderDecoderConfig, VisionEncoderDecoderModel

	>>> # Initializing a ViT & BERT style configuration
	>>> config_encoder = ViTConfig()
	>>> config_decoder = BertConfig()

	>>> config = VisionEncoderDecoderConfig.from_encoder_decoder_configs(config_encoder, config_decoder)

	>>> # Initializing a ViTBert model (with random weights) from a ViT & google-bert/bert-base-uncased style configurations
	>>> model = VisionEncoderDecoderModel(config=config)

	>>> # Accessing the model configuration
	>>> config_encoder = model.config.encoder
	>>> config_decoder = model.config.decoder
	>>> # set decoder config to causal lm
	>>> config_decoder.is_decoder = True
	>>> config_decoder.add_cross_attention = True

	>>> # Saving the model, including its configuration
	>>> model.save_pretrained("my-model")

	>>> # loading model and config from pretrained folder
	>>> encoder_decoder_config = VisionEncoderDecoderConfig.from_pretrained("my-model")
	>>> model = VisionEncoderDecoderModel.from_pretrained("my-model", config=encoder_decoder_config)
	```"""

	model_type = "vision-encoder-decoder"
	is_composition = True

	def __init__(self, **kwargs):
	super().__init__(**kwargs)
	if "encoder" not in kwargs or "decoder" not in kwargs:
	raise ValueError(
	f"A configuraton of type {self.model_type} cannot be instantiated because "
	f"not both `encoder` and `decoder` sub-configurations are passed, but only {kwargs}"
	)

	encoder_config = kwargs.pop("encoder")
	encoder_model_type = encoder_config.pop("model_type")
	decoder_config = kwargs.pop("decoder")
	decoder_model_type = decoder_config.pop("model_type")

	self.encoder = AutoConfig.for_model(encoder_model_type, **encoder_config)
	self.decoder = AutoConfig.for_model(decoder_model_type, **decoder_config)
	self.is_encoder_decoder = True

	@classmethod
	def from_encoder_decoder_configs(
	cls, encoder_config: PretrainedConfig, decoder_config: PretrainedConfig, **kwargs
	) -> PretrainedConfig:
	r"""
	Instantiate a [`VisionEncoderDecoderConfig`] (or a derived class) from a pre-trained encoder model
	configuration and decoder model configuration.

	Returns:
	[`VisionEncoderDecoderConfig`]: An instance of a configuration object
	"""
	logger.info("Setting `config.is_decoder=True` and `config.add_cross_attention=True` for decoder_config")
	decoder_config.is_decoder = True
	decoder_config.add_cross_attention = True

	return cls(encoder=encoder_config.to_dict(), decoder=decoder_config.to_dict(), **kwargs)

	class SpeechEncoderDecoderConfig(PretrainedConfig):
	r"""
	[`SpeechEncoderDecoderConfig`] is the configuration class to store the configuration of a
	[`SpeechEncoderDecoderModel`]. It is used to instantiate an Encoder Decoder model according to the specified
	arguments, defining the encoder and decoder configs.

	Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
	documentation from [`PretrainedConfig`] for more information.

	Args:
	kwargs (optional):
	Dictionary of keyword arguments. Notably:

	- encoder ([`PretrainedConfig`], optional) -- An instance of a configuration object that defines
	the encoder config.
	- decoder ([`PretrainedConfig`], optional) -- An instance of a configuration object that defines
	the decoder config.

	Examples:

	```python
	>>> from transformers import BertConfig, Wav2Vec2Config, SpeechEncoderDecoderConfig, SpeechEncoderDecoderModel

	>>> # Initializing a Wav2Vec2 & BERT style configuration
	>>> config_encoder = Wav2Vec2Config()
	>>> config_decoder = BertConfig()

	>>> config = SpeechEncoderDecoderConfig.from_encoder_decoder_configs(config_encoder, config_decoder)

	>>> # Initializing a Wav2Vec2Bert model from a Wav2Vec2 & google-bert/bert-base-uncased style configurations
	>>> model = SpeechEncoderDecoderModel(config=config)

	>>> # Accessing the model configuration
	>>> config_encoder = model.config.encoder
	>>> config_decoder = model.config.decoder
	>>> # set decoder config to causal lm
	>>> config_decoder.is_decoder = True
	>>> config_decoder.add_cross_attention = True

	>>> # Saving the model, including its configuration
	>>> model.save_pretrained("my-model")

	>>> # loading model and config from pretrained folder
	>>> encoder_decoder_config = SpeechEncoderDecoderConfig.from_pretrained("my-model")
	>>> model = SpeechEncoderDecoderModel.from_pretrained("my-model", config=encoder_decoder_config)
	```"""

	model_type = "speech-encoder-decoder"
	is_composition = True

	def __init__(self, **kwargs):
	super().__init__(**kwargs)
	if "encoder" not in kwargs or "decoder" not in kwargs:
	raise ValueError(
	f"A configuraton of type {self.model_type} cannot be instantiated because not both `encoder` and"
	f" `decoder` sub-configurations are passed, but only {kwargs}"
	)

	encoder_config = kwargs.pop("encoder")
	encoder_model_type = encoder_config.pop("model_type")
	decoder_config = kwargs.pop("decoder")
	decoder_model_type = decoder_config.pop("model_type")

	self.encoder = AutoConfig.for_model(encoder_model_type, **encoder_config)
	self.decoder = AutoConfig.for_model(decoder_model_type, **decoder_config)
	self.is_encoder_decoder = True

	@classmethod
	def from_encoder_decoder_configs(
	cls, encoder_config: PretrainedConfig, decoder_config: PretrainedConfig, **kwargs
	) -> PretrainedConfig:
	r"""
	Instantiate a [`SpeechEncoderDecoderConfig`] (or a derived class) from a pre-trained encoder model
	configuration and decoder model configuration.

	Returns:
	[`SpeechEncoderDecoderConfig`]: An instance of a configuration object
	"""
	logger.info("Setting `config.is_decoder=True` and `config.add_cross_attention=True` for decoder_config")
	decoder_config.is_decoder = True
	decoder_config.add_cross_attention = True

	return cls(encoder=encoder_config.to_dict(), decoder=decoder_config.to_dict(), **kwargs)

	class MistralConfig(PretrainedConfig):
	is_composition = True

	r"""
	This is the configuration class to store the configuration of a [`MistralModel`]. It is used to instantiate an
	Mistral model according to the specified arguments, defining the model architecture. Instantiating a configuration
	with the defaults will yield a similar configuration to that of the Mistral-7B-v0.1 or Mistral-7B-Instruct-v0.1.

	[mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1)
	[mistralai/Mistral-7B-Instruct-v0.1](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1)

	Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
	documentation from [`PretrainedConfig`] for more information.


	Args:
	vocab_size (`int`, optional, defaults to 32000):
	Vocabulary size of the Mistral model. Defines the number of different tokens that can be represented by the
	`inputs_ids` passed when calling [`MistralModel`]
	hidden_size (`int`, optional, defaults to 4096):
	Dimension of the hidden representations.
	intermediate_size (`int`, optional, defaults to 14336):
	Dimension of the MLP representations.
	num_hidden_layers (`int`, optional, defaults to 32):
	Number of hidden layers in the Transformer encoder.
	num_attention_heads (`int`, optional, defaults to 32):
	Number of attention heads for each attention layer in the Transformer encoder.
	num_key_value_heads (`int`, optional, defaults to 8):
	This is the number of key_value heads that should be used to implement Grouped Query Attention. If
	`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
	`num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
	converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
	by meanpooling all the original heads within that group. For more details checkout [this
	paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `8`.
	hidden_act (`str` or `function`, optional, defaults to `"silu"`):
	The non-linear activation function (function or string) in the decoder.
	max_position_embeddings (`int`, optional, defaults to `4096*32`):
	The maximum sequence length that this model might ever be used with. Mistral's sliding window attention
	allows sequence of up to 4096*32 tokens.
	initializer_range (`float`, optional, defaults to 0.02):
	The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
	rms_norm_eps (`float`, optional, defaults to 1e-06):
	The epsilon used by the rms normalization layers.
	use_cache (`bool`, optional, defaults to `True`):
	Whether or not the model should return the last key/values attentions (not used by all models). Only
	relevant if `config.is_decoder=True`.
	pad_token_id (`int`, optional):
	The id of the padding token.
	bos_token_id (`int`, optional, defaults to 1):
	The id of the "beginning-of-sequence" token.
	eos_token_id (`int`, optional, defaults to 2):
	The id of the "end-of-sequence" token.
	tie_word_embeddings (`bool`, optional, defaults to `False`):
	Whether the model's input and output word embeddings should be tied.
	rope_theta (`float`, optional, defaults to 10000.0):
	The base period of the RoPE embeddings.
	sliding_window (`int`, optional, defaults to 4096):
	Sliding window attention window size. If not specified, will default to `4096`.
	attention_dropout (`float`, optional, defaults to 0.0):
	The dropout ratio for the attention probabilities.

	```python
	>>> from transformers import MistralModel, MistralConfig

	>>> # Initializing a Mistral 7B style configuration
	>>> configuration = MistralConfig()

	>>> # Initializing a model from the Mistral 7B style configuration
	>>> model = MistralModel(configuration)

	>>> # Accessing the model configuration
	>>> configuration = model.config
	```"""

	model_type = ["mistral","speech-encoder-decoder","image-encoder-decoder","mistral-encoder-decoder"]
	# model_type = "mistral-encoder-decoder"
	keys_to_ignore_at_inference = ["past_key_values"]

	def __init__(
	self,
	vocab_size=32000,
	hidden_size=4096,
	intermediate_size=14336,
	num_hidden_layers=32,
	num_attention_heads=32,
	num_key_value_heads=8,
	hidden_act="silu",
	max_position_embeddings=4096 * 32,
	initializer_range=0.02,
	rms_norm_eps=1e-6,
	use_cache=True,
	pad_token_id=None,
	bos_token_id=1,
	eos_token_id=2,
	tie_word_embeddings=False,
	sliding_window=4096,
	attention_dropout=0.0,

	# for yarn Later
	rope_theta=10000.0,
	rope_scaling=None,
	# for thought generation Later
	max_thoughts=16,
	max_temperature=10,
	complexity_factor = 0.5,
	merged_talk_heads=True,
	merged_lm_and_talk_heads=False,
	merged_lm_and_think_heads=True,
	use_concat_talk_head=True,
	use_shallow_think=True,
	use_shallow_talk=False,
	use_complex_think_head=False,
	use_complex_talk_head=True,
	use_weighted_talk_head=True,
	hidden_dropout_prob=0.00,

	**kwargs,
	):
	super().__init__(
	pad_token_id=pad_token_id,
	bos_token_id=bos_token_id,
	eos_token_id=eos_token_id,
	tie_word_embeddings=tie_word_embeddings,
	**kwargs,
	)

	self.vocab_size = vocab_size
	self.max_position_embeddings = max_position_embeddings
	self.hidden_size = hidden_size
	self.intermediate_size = intermediate_size
	self.num_hidden_layers = num_hidden_layers
	self.num_attention_heads = num_attention_heads
	self.sliding_window = sliding_window

	# for backward compatibility
	if num_key_value_heads is None:
	num_key_value_heads = num_attention_heads

	self.num_key_value_heads = num_key_value_heads
	self.hidden_act = hidden_act
	self.initializer_range = initializer_range
	self.rms_norm_eps = rms_norm_eps
	self.use_cache = use_cache
	self.attention_dropout = attention_dropout
	# yarn
	self.rope_scaling = rope_scaling
	self._rope_scaling_validation()
	self.rope_theta = rope_theta
	#Thought gen
	self.max_thoughts = max_thoughts
	self.complexity_factor = complexity_factor
	self.max_temperature = max_temperature
	self.merged_talk_heads = merged_talk_heads
	self.merged_lm_and_talk_heads = merged_lm_and_talk_heads
	self.merged_lm_and_think_heads = merged_lm_and_think_heads
	self.use_concat_talk_head = use_concat_talk_head
	self.use_shallow_think = use_shallow_think
	self.use_shallow_talk = use_shallow_talk
	self.use_complex_think_head = use_complex_think_head
	self.use_complex_talk_head = use_complex_talk_head
	self.use_weighted_talk_head = use_weighted_talk_head
	self.hidden_dropout_prob = hidden_dropout_prob
	#Encoder Decoder - Currently only a single EncoderDecoder is supported -Later will be eXpanded to suport both models
	if "encoder" not in kwargs or "decoder" not in kwargs:
	raise ValueError(
	f"A configuraton of type {self.model_type} cannot be instantiated because "
	f"not both `encoder` and `decoder` sub-configurations are passed, but only {kwargs}"
	)

	encoder_config = kwargs.pop("encoder")
	encoder_model_type = encoder_config.pop("model_type")
	decoder_config = kwargs.pop("decoder")
	decoder_model_type = decoder_config.pop("model_type")

	self.encoder = AutoConfig.for_model(encoder_model_type, **encoder_config)
	self.decoder = AutoConfig.for_model(decoder_model_type, **decoder_config)
	self.is_encoder_decoder = True

	@classmethod
	def from_encoder_decoder_configs(
	cls, encoder_config: PretrainedConfig, decoder_config: PretrainedConfig, **kwargs
	) -> PretrainedConfig:
	r"""
	Instantiate a [`SpeechEncoderDecoderConfig`] (or a derived class) from a pre-trained encoder model
	configuration and decoder model configuration.

	Returns:
	[`SpeechEncoderDecoderConfig`]: An instance of a configuration object
	"""
	logger.info("Setting `config.is_decoder=True` and `config.add_cross_attention=True` for decoder_config")
	decoder_config.is_decoder = True
	decoder_config.add_cross_attention = True

	return cls(encoder=encoder_config.to_dict(), decoder=decoder_config.to_dict(), **kwargs)

	def _rope_scaling_validation(self):
	"""
	Validate the `rope_scaling` configuration.
	"""
	if self.rope_scaling is None:
	return

	if not isinstance(self.rope_scaling, dict):
	raise ValueError(
	"`rope_scaling` must be a dictionary, "
	f"got {self.rope_scaling}"
	)
	rope_scaling_type = self.rope_scaling.get("type", None)
	rope_scaling_factor = self.rope_scaling.get("factor", None)
	if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic", "yarn", "dynamic-yarn"]:
	raise ValueError(
	f"`rope_scaling`'s name field must be one of ['linear', 'dynamic', 'yarn', 'dynamic-yarn'], got {rope_scaling_type}"
	)
	if rope_scaling_factor is None or not isinstance(rope_scaling_factor, float) or rope_scaling_factor <= 1.0:
	raise ValueError(f"`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}")
	if rope_scaling_type == "yarn" or rope_scaling_type == "dynamic-yarn":
	original_max_position_embeddings = self.rope_scaling.get("original_max_position_embeddings", None)
	if original_max_position_embeddings is None or not isinstance(original_max_position_embeddings, int):
	raise ValueError(f"`rope_scaling.original_max_position_embeddings` must be set to an int when using yarn, and dynamic-yarn")