Added the alpha version of Corpus2GPT

inference/inference.py

@@ -0,0 +1,99 @@
+import keras
+import numpy as np
+from .sampling_strategies.sample_random import *
+
+class Generative_inference:
+    """ 
+    This class facilitates text generation by utilizing a provided Keras model, 
+    tokenizer, and search strategy. It allows for the generation of text based 
+    on an initial prompt.
+
+    Example:
+        ```
+        >>> inference = Generative_inference(model = model,
+        >>>                          tokenizer = tokenizer,
+        >>>                          search_strategy=random_sampling_strategy)
+        >>> inference.generate("Hello World")
+         ⁇  ⁇  ⁇  ⁇  ⁇  ⁇  ⁇  ⁇  ⁇  Hello WorldAr things sayingWhen ruby...
+        ```
+    
+    """
+    def __init__(self,
+                 model,
+                 tokenizer,
+                 search_strategy=random_sampling_strategy,
+                 prompt="Hello World",
+                 input_len=64,
+                 padding_token=0,
+                 **kwargs
+                 ):
+        """
+        Constructor for Generative_inference class.
+
+        Args:
+            model: A Keras model used for text generation.
+            tokenizer: Tokenizer used to encode and decode text.
+            search_strategy: Strategy used for searching tokens during generation. Default is `random_sampling_strategy`
+            prompt (str): The initial prompt for text generation. Default is "Hello World".
+            input_len (int): Length of the input tokens. Default is 64.
+            padding_token (int): Token used for padding. Default is 0.
+        """
+        self.search_strategy = search_strategy
+        self.kwargs = kwargs
+        self.model = model
+        self.tokenizer = tokenizer
+        self.prompt = prompt
+        self.padding_token = padding_token
+        self.input_len = input_len
+
+    def generate(self,
+                 prompt=None,
+                 generate_limit=50,
+                 **kwargs):
+        """
+        Generate text based on the provided prompt.
+
+        Args:
+            prompt (str): The prompt for text generation. If not provided, uses the default prompt.
+            generate_limit (int): Maximum number of tokens to generate. Default is 50.
+            **kwargs: Additional keyword arguments to be passed to the search_strategy.
+
+        Returns:
+            str: Generated text.
+        """
+
+        if prompt is None:
+            prompt = self.prompt
+        
+        prompt_tokens = self.tokenizer.tokenizer.encode_as_ids(prompt)
+
+        input_prompt_token_len = len(prompt_tokens)
+
+        if len(prompt_tokens) > self.input_len:
+            prompt_tokens = prompt_tokens[:self.input_len]
+        elif len(prompt_tokens) < self.input_len:
+            prompt_tokens = [self.padding_token] * (self.input_len - len(prompt_tokens)) + prompt_tokens
+        else:
+            pass
+
+        model_input = keras.ops.convert_to_tensor(prompt_tokens)
+        model_input = keras.ops.reshape(model_input, (1, self.input_len))
+
+        gen_len = 0
+        while gen_len < generate_limit:
+
+            gen_len += 1
+
+            model_output = self.model(model_input)
+            output_token = self.search_strategy(outputs=model_output, pos_num=-1, **self.kwargs)
+            model_input = keras.ops.convert_to_numpy(model_input)
+            model_input = np.concatenate((model_input, [[output_token]]), -1)
+            model_input = model_input[:, 1 :]
+            # model_input = keras.ops.convert_to_tensor(model_input)
+        
+        model_input = keras.ops.reshape(model_input, (self.input_len,))
+        model_input = keras.ops.convert_to_numpy(model_input)
+
+        model_output = self.tokenizer.tokenizer.decode_ids(model_input.tolist())
+
+        return model_output

inference/sampling_strategies/sample_random.py

@@ -0,0 +1,16 @@
+import keras
+import numpy as np
+
+def random_sampling_strategy(outputs, pos_num=0, k_value=3):
+
+    if len(keras.ops.shape(outputs)) == 3:
+        outputs = outputs[0][pos_num]
+    else:
+        outputs = outputs[pos_num]
+    
+    values, indices = keras.ops.top_k(outputs, k=k_value)
+    values = keras.ops.softmax(values, -1)
+    values = keras.ops.convert_to_numpy(values)
+    indices = keras.ops.convert_to_numpy(indices)
+
+    return np.random.choice(indices, p=values)

inference/scale_utils.py

@@ -0,0 +1,150 @@
+import numpy as np
+from models.GPT import build_GPT
+
+# Utils to work with estimation functions
+
+def normalize_list(numbers):
+    """
+    Normalizes a list of numbers to the range [0, 1].
+    
+    Args:
+        numbers (list of numeric): List of numbers to be normalized.
+        
+    Returns:
+        list of float: Normalized list of numbers.
+    """
+    min_val = min(numbers)
+    max_val = max(numbers)
+    normalized = [(x - min_val) / (max_val - min_val) for x in numbers]
+    return normalized
+
+
+def estimate_optimal_ratios_from_models(model_configs, 
+                                       train_seq_len,
+                                       x_train,
+                                       y_train,
+                                       max_epochs,
+                                       batch_size):
+    """
+    Estimate the optimal ratios of model size and number of training tokens from FLOP counts.
+
+    Args:
+    - model_configs (list): List of tuples representing model configurations.
+                            Each tuple contains parameters for building the model.
+    - train_seq_len (list): List of integers representing different numbers of training sequences.
+    - x_train (numpy array): Input data for training.
+    - y_train (numpy array): Target data for training.
+    - max_epochs (int): Maximum number of epochs for training.
+    - batch_size (int): Batch size for training.
+
+    Returns:
+    - flops (numpy array): Array of FLOP counts for each experiment.
+    - loss_history (numpy array): Array of loss histories for each experiment.
+    - model_params (numpy array): Array of total model parameters for each experiment.
+    """
+
+    total_models = len(model_configs)
+    total_seq_len = len(train_seq_len)
+
+    print('Total Number of Experiments: ' + str(total_models * total_seq_len))
+
+    experiment_number = 0
+    _flops = []
+    _loss_history = []
+    _model_params = []
+    for model_config in model_configs:
+        for seq_len in train_seq_len:
+            experiment_number += 1
+            print('Train Number: ' + str(experiment_number))
+
+            # Build the model and calculate FLOPs
+            GPT, flops = build_GPT(*model_config)
+            
+            # Train the model
+            history = GPT.fit(x_train[:seq_len], y_train[:seq_len], batch_size=batch_size, epochs=max_epochs)
+            
+            # Count model parameters
+            model_params = GPT.count_params()
+
+            # Extract loss history
+            loss_history = history.history['loss']
+            
+            # Store results
+            _flops.append(flops*seq_len*max_epochs)
+            _loss_history.append(loss_history)
+            _model_params.append(model_params)
+    
+    return (np.array(_flops), np.array(_loss_history), np.array(_model_params))
+
+import numpy as np
+
+def estimate_optimal_ratios_from_flops(flop_list,
+                                       input_len,
+                                       num_heads,
+                                       head_dims,
+                                       num_decoders,
+                                       fc_dim_factor,
+                                       vocab_size,
+                                       dropout_rate,
+                                       x_train,
+                                       y_train,
+                                       trials_per_flop=2,
+                                       batch_size=32):
+    """
+    Estimates optimal ratios of various model parameters based on FLOP count.
+
+    Args:
+        flop_list (list): List of FLOP counts to estimate optimal ratios for.
+        input_len (int): Length of the input sequence.
+        num_heads (tuple): Tuple containing the minimum and maximum values for the number of attention heads.
+        head_dims (tuple): Tuple containing the minimum and maximum values for the dimensionality of attention heads.
+        num_decoders (int): Number of decoder layers.
+        fc_dim_factor (int): Factor to determine the dimensionality of fully connected layers.
+        vocab_size (int): Size of the vocabulary.
+        dropout_rate (float): Dropout rate.
+        x_train (numpy.ndarray): Training input data.
+        y_train (numpy.ndarray): Training target data.
+        trials_per_flop (int, optional): Number of trials per FLOP count. Defaults to 2.
+        batch_size (int, optional): Batch size for training. Defaults to 32.
+
+    Returns:
+        tuple: Tuple containing loss history, FLOP history, and number of parameters for each trial.
+    """
+
+    loss_history = []
+    flop_history = []
+    parameters = []
+
+    for flop in flop_list:
+        for _ in range(trials_per_flop):
+            f_num_heads = np.random.randint(num_heads[0], num_heads[1])
+            f_head_dims = np.random.randint(head_dims[0], head_dims[1])
+            f_embed_dim = f_num_heads * f_head_dims
+            f_num_decoders = np.random.randint(1, num_decoders)
+            f_fc_dim_factor = np.random.randint(1, fc_dim_factor)
+
+            args = (input_len,
+                    vocab_size,
+                    f_embed_dim,
+                    f_num_decoders,
+                    dropout_rate,
+                    f_num_heads,
+                    f_head_dims,
+                    f_fc_dim_factor
+                    )
+
+            GPT, flop_per_inference = build_GPT(*args)  # Assuming build_GPT is defined elsewhere
+            print(GPT.summary())
+
+            epochs = flop // flop_per_inference
+            if epochs <= 0:
+                raise Exception('The provided FLOP count is too small: ' + str(flop) + ' is too small')
+
+            history = GPT.fit(x_train, y_train, batch_size=batch_size, epochs=epochs)
+
+            loss_history.append(history.history['loss'])
+            flop_history.append(flop*batch_size*epochs)
+            parameters.append(GPT.count_params())
+
+    return loss_history, flop_history, parameters
+

models/GPT.py

@@ -0,0 +1,167 @@
+import keras
+from tokenizer.tokenizer import *
+from models.attention import *
+from models.decoder import *
+from models.embeddings import *
+
+def FLOP(input_len, vocab_size, embed_dim, num_heads, num_decoders, fc_dim_factor):
+    """ 
+    Calculate total number of FLOPs, see Chinchilla 
+    paper Appendix F as reference: https://arxiv.org/pdf/2203.15556.pdf
+
+    Copied from: https://github.com/karpathy/nanoGPT/blob/master/scaling_laws.ipynb
+    """ 
+    key_size = embed_dim // num_heads
+
+    # embeddings
+    embeddings = 2 * input_len * vocab_size * embed_dim
+
+    # attention
+    # key, query, value projections
+    attention = 2 * 3 * input_len * embed_dim * (key_size * num_heads)
+    # key @ query logits
+    attlogits = 2 * input_len * input_len * (key_size * num_heads)
+    # softmax
+    attsoftmax = 3 * num_heads * input_len * input_len # 3* is for subtract (max), exp, divide (?)
+    # softmax @ value reductions
+    attvalue = 2 * input_len * input_len * (key_size * num_heads)
+    # final linear
+    attlinear = 2 * input_len * (key_size * num_heads) * embed_dim
+    att = attention + attlogits + attsoftmax + attvalue + attlinear
+    # feed forward
+    dense = 2 * input_len * (embed_dim * embed_dim*fc_dim_factor + embed_dim * embed_dim*fc_dim_factor)
+
+    # logits
+    logits = 2 * input_len * embed_dim * vocab_size
+    
+    # this is what you'd expect:
+    # forward_flops = embeddings + num_decoders * (att + dense) + logits
+    # but:
+    # per author correspondence apparently there is typo in the paper,
+    # they do not count embeddings and logits to repro table 4. So instead:
+    forward_flops = num_decoders * (att + dense)
+    backward_flops = 2 * forward_flops # as in Kaplan et al. 2020
+    total_flops = forward_flops + backward_flops
+
+    return total_flops
+
+class GPT(keras.layers.Layer):
+    """
+    GPT (Generative Pre-trained Transformer) layer.
+
+    This layer implements the architecture of the GPT model, which consists of multiple decoder layers followed
+    by a linear mapping head for language modeling.
+
+    Parameters:
+        decoder (class): Class representing the decoder layer of the Transformer model.
+        attention (class): Class representing the attention mechanism used in the decoder layer.
+        embeddings (class): Class representing the token embeddings.
+        pos_embeddings (class): Class representing the positional embeddings.
+        embedding_size (int): Size of the token embeddings. Default is 1280.
+        vocab_size (int): Size of the vocabulary. Default is 8008.
+        input_len (int): Length of the input sequence. Default is 64.
+        num_decoders (int): Number of decoder layers in the GPT model. Default is 10.
+
+    Attributes:
+        num_decoders (int): Number of decoder layers in the GPT model.
+        decoders (list): List of decoder layer instances.
+        embeddings (keras.layers.Layer): Token embeddings layer instance.
+        pos_embeddings (keras.layers.Layer): Positional embeddings layer instance.
+        lm_head (keras.layers.Dense): Dense layer for language modeling.
+    """
+
+    def __init__(self, decoder, embeddings, pos_embeddings = None, embedding_size=1280, vocab_size=8008, input_len=64, num_decoders=5, dropout_rate=0.1, num_heads = 32, head_dims = 40, fc_dim_factor = 5):
+        """
+        Initializes the GPT layer.
+
+        Args:
+            decoder (class): Class representing the decoder layer of the Transformer model.
+            attention (class): Class representing the attention mechanism used in the decoder layer.
+            embeddings (class): Class representing the token embeddings.
+            pos_embeddings (class): Class representing the positional embeddings.
+            embedding_size (int): Size of the token embeddings. Default is 1280.
+            vocab_size (int): Size of the vocabulary. Default is 8008.
+            input_len (int): Length of the input sequence. Default is 64.
+            num_decoders (int): Number of decoder layers in the GPT model. Default is 10.
+        """
+        super().__init__()
+
+        self.num_decoders = num_decoders
+        self.decoders = []
+        for _ in range(self.num_decoders):
+            self.decoders.append(decoder(dropout_rate, num_heads, head_dims, fc_dim_factor, input_len = input_len))
+        
+        self.embeddings = embeddings(input_len, vocab_size+1, embed_dim=embedding_size)
+
+        self.lm_head = keras.layers.Dense(vocab_size)
+
+        self._config = {'decoder' : decoder, 'embeddings': embeddings, 'pos_embeddings': pos_embeddings, 'embedding_size': embedding_size, 'vocab_size': vocab_size, 'input_len': input_len, 'num_decoders': num_decoders}
+
+
+    def call(self, inputs):
+        """
+        Executes the forward pass of the GPT layer.
+
+        Args:
+            inputs: Input tensor representing the token indices.
+
+        Returns:
+            Tensor: Output tensor representing the logits for language modeling.
+        """
+        x = inputs
+
+        x = self.embeddings(x)
+
+        for decoder in self.decoders:
+            x = decoder(x)
+        
+        x = self.lm_head(x)
+
+        return x
+
+
+
+def build_GPT(input_len,
+              vocab_size,
+              embed_dim,
+              num_decoders,
+              dropout_rate,
+              num_heads,
+              head_dims,
+              fc_dim_factor,
+              optimizer='adam'
+              ):
+    """
+    Builds a GPT (Generative Pre-trained Transformer) model.
+
+    Parameters:
+        input_len (int): The length of the input sequence.
+        vocab_size (int): The size of the vocabulary.
+        embed_dim (int): The dimensionality of the token embeddings.
+        num_decoders (int): The number of decoder layers.
+        dropout_rate (float): The dropout rate to apply within the model.
+        num_heads (int): The number of attention heads in each decoder layer.
+        head_dims (int): The dimensionality of each attention head.
+        fc_dim_factor (int): The factor to determine the dimensionality
+                             of the feedforward network within each decoder layer.
+        optimizer (str, optional): The optimizer to use for training. 
+                                   Defaults to 'adam'.
+
+    Returns:
+        tuple: A tuple containing the GPT model and the total number of floating-point operations (FLOPs).
+
+    """
+    GPT = keras.Sequential()
+    GPT.add(keras.Input(shape=(input_len,)))
+    GPT.add(TokenAndPositionEmbedding(input_len, vocab_size, embed_dim))
+    for _ in range(num_decoders):
+        GPT.add(Decoder(dropout_rate, num_heads, head_dims, fc_dim_factor, input_len))
+    GPT.add(keras.layers.Dense(vocab_size+1))
+
+    loss_fn = keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+    GPT.compile(optimizer=optimizer, loss=[loss_fn])
+
+    # Calculate the total number of floating-point operations              
+    flops = FLOP(input_len, vocab_size, embed_dim, num_heads, num_decoders, fc_dim_factor)
+    return GPT, flops
+

models/attention.py

@@ -0,0 +1,222 @@
+import keras
+import numpy as np
+
+
+class Attention(keras.layers.Layer):
+    """
+    Multihead attention layer.
+
+    This layer performs multihead attention on input sequences
+    `(key, query, value)`. It splits the input into multiple heads,
+    applies attention mechanism independently to each head,
+    and concatenates the outputs.
+
+    Parameters:
+        head_dims (int): Dimensionality of each head.
+        num_heads (int): Number of attention heads.
+        dropout (float): Dropout rate to apply within the
+                         attention mechanism.
+
+    Usage:
+    ```
+        attention = Attention(head_dims=40, num_heads=32, dropout=0.2)
+        output, cache = attention([key, query, value])
+    ```
+    """
+
+    def __init__(self, head_dims=40, num_heads=32, dropout=0.2):
+        """
+        Initializes the multihead attention layer.
+
+        Args:
+            head_dims (int): Dimensionality of each head.
+            num_heads (int): Number of attention heads.
+            dropout (float): Dropout rate to apply within
+                             the attention mechanism.
+        """
+        super().__init__()
+
+        self.head_dims = head_dims
+        self.num_heads = num_heads
+        self.dropout = dropout
+        self.dense_units = self.head_dims * self.num_heads
+
+        self.key = keras.layers.Dense(self.dense_units)
+        self.query = keras.layers.Dense(self.dense_units)
+        self.value = keras.layers.Dense(self.dense_units)
+        self.out = keras.layers.Dense(self.dense_units)
+        self.norm = keras.layers.LayerNormalization(-1)
+        self.dropout = keras.layers.Dropout(self.dropout)
+
+        self.q_norm_factor = 1/np.sqrt(self.num_heads * self.head_dims)
+
+        self._config = {'head_dims': head_dims, 'num_heads': num_heads, 'dropout': dropout}
+
+    def generate_mask(self, num_words):
+        """
+        Generates a triangular mask to be applied 
+        to attention scores to prevent attending to 
+        future positions.
+
+        Args:
+            num_words (int): Number of words in the 
+            sequence.
+
+        Returns:
+            tf.Tensor: Triangular mask tensor.
+        """
+        tensor = np.full((num_words, num_words), np.inf)  # Initialize tensor with infinity
+        for i in range(num_words):
+            tensor[i, :i + 1] = 0
+        return keras.ops.convert_to_tensor(tensor, dtype="float32")
+
+    def _shape(self, tensor):
+        """
+        Reshapes the input tensor for multihead attention
+        computations.
+
+        Args:
+            tensor (tf.Tensor): Input tensor.
+
+        Returns:
+            tf.Tensor: Reshaped tensor.
+        """
+        bsz = keras.ops.shape(tensor)[0]
+        tensor = keras.ops.reshape(tensor, (bsz, -1, self.num_heads, self.head_dims))
+        tensor = keras.ops.transpose(tensor, (0, 2, 1, 3))
+        return tensor
+
+    def call(self, inputs, use_cache=None):
+        """
+        Forward pass of the multihead attention layer.
+
+        Args:
+            inputs (list): List containing key, query, 
+              	           and value tensors.
+            use_cache (tuple): Cache from previous 
+                                attention operation (unsupported).
+
+        Returns:
+            tf.Tensor: Output tensor.
+            tuple: Cache for subsequent attention operations.
+        """
+        k, q, v = inputs
+
+        if use_cache is None:
+            k = self.key(k)
+            q = self.query(q)
+            v = self.value(v)
+
+            k, q, v = self._shape(k), self._shape(q), self._shape(v)
+
+        else:
+            raise NotImplementedError("`use_cache` argument is not supported yet!")
+
+        cache = (k, q, v)
+
+        kq = keras.ops.einsum('bijk, bilk -> bij', k, q)
+        kq *= self.q_norm_factor
+
+        num_words = keras.ops.shape(kq)[-1]
+        #num_words = 64
+        bsz = keras.ops.shape(kq)[0]
+
+        kq = keras.ops.reshape(kq, (bsz, 1, -1, num_words))
+
+        kq_copy = keras.ops.copy(kq)
+
+        for counter in range(num_words - 1):
+            kq = keras.ops.append(kq, kq_copy, 1)
+
+        mask = self.generate_mask(num_words)
+        kq = keras.ops.transpose(kq, (0, 2, 1, 3))
+        kq = kq - mask
+        kq = keras.ops.transpose(kq, (0, 2, 1, 3))
+        kq = keras.ops.softmax(kq, -1)
+        kqv = keras.ops.einsum('bijk, bjkl -> bijl', kq, v)
+        kqv = keras.ops.reshape(kqv, (bsz, num_words, -1))
+        kqv = self.norm(kqv)
+        kqv = self.dropout(kqv)
+        kqv = self.out(kqv)
+
+        return kqv, cache
+
+
+
+class AttentionTrain(keras.layers.Layer):
+    def _shape(self, tensor):
+      """
+      Reshapes the input tensor for multihead attention
+      computations.
+
+      Args:
+          tensor (tf.Tensor): Input tensor.
+
+      Returns:
+          tf.Tensor: Reshaped tensor.
+      """
+      bsz = keras.ops.shape(tensor)[0]
+      tensor = keras.ops.reshape(tensor, (bsz, -1, self.num_heads, self.head_dims))
+      tensor = keras.ops.transpose(tensor, (0, 2, 1, 3))
+      return tensor
+    
+    def generate_mask(self, num_words):
+        """
+        Generates a triangular mask to be applied 
+        to attention scores to prevent attending to 
+        future positions.
+
+        Args:
+            num_words (int): Number of words in the 
+            sequence.
+
+        Returns:
+            tf.Tensor: Triangular mask tensor.
+        """
+        tensor = np.full((num_words, num_words), np.inf)  # Initialize tensor with infinity
+        for i in range(num_words):
+            tensor[i, :i + 1] = 0
+        return keras.ops.convert_to_tensor(tensor, dtype="float32")
+
+
+    def __init__(self, num_heads, head_dims, dropout = 0.2, input_len = 64):
+        super().__init__()
+        self.num_heads = num_heads
+        self.head_dims = head_dims
+
+        self.k = keras.layers.Dense(self.num_heads * self.head_dims)
+        self.q = keras.layers.Dense(self.num_heads * self.head_dims)
+        self.v = keras.layers.Dense(self.num_heads * self.head_dims)
+        self.out = keras.layers.Dense(self.num_heads * self.head_dims)
+
+        self.q_norm = 1/keras.ops.sqrt(self.num_heads * self.head_dims)
+        self.mask = self.generate_mask(input_len)
+
+        self.dropout = keras.layers.Dropout(dropout)
+
+    
+    def call(self, inputs):
+
+        k = self.k(inputs)
+        q = self.q(inputs)
+        v = self.v(inputs)
+
+        k, q, v = self._shape(k), self._shape(q), self._shape(v)
+
+        # (b, head, k_token, dims), (b, head, q_token, dims) -> (b, head, q_token, k_token)
+        kq = keras.ops.einsum('bijk, bilk -> bilj', k, q)
+        kq *= self.q_norm
+        kq -= self.mask
+        kq = self.dropout(kq)
+        kq = keras.ops.softmax(kq, -1)
+        
+        # (b, head, q_token, k_token), (b, head, k_token, dims) -> (b, head, q_token, dims)
+        kqv = keras.ops.einsum('bilj, bijk -> bilk', kq, v)
+
+        kqv = keras.ops.transpose(kqv, (0, 2, 1, 3))
+
+        bsz = keras.ops.shape(v)[0]
+        kqv = keras.ops.reshape(kqv, (bsz, -1, self.num_heads * self.head_dims))
+        kqv = self.out(kqv)
+
+        return kqv

models/decoder.py

@@ -0,0 +1,56 @@
+import keras
+from models.attention import Attention, AttentionTrain
+
+class Decoder(keras.layers.Layer):
+    """
+    Decoder layer in a Transformer model architecture.
+
+    This layer implements the decoder component of the Transformer model, which is responsible for generating
+    the output sequence based on the encoded input sequence and previously generated output tokens.
+
+    Parameters:
+        attention (keras.layers.Layer): Attention layer for the decoder.
+        dropout_rate (float): Dropout rate applied to the outputs of each sub-layer. Default is 0.2.
+    """
+
+    def __init__(self, dropout_rate=0.2, num_heads = 32, head_dims = 40, fc_dim_factor = 5, input_len = 64):
+        """
+        Initializes the Decoder layer.
+
+        Args:
+            attention (keras.layers.Layer): Attention layer for the decoder.
+            dropout_rate (float): Dropout rate applied to the outputs of each sub-layer. Default is 0.2.
+        """
+        super().__init__()
+
+        # Layer Normalization for the first sub-layer
+        self.norm1 = keras.layers.LayerNormalization(epsilon=1e-9)
+
+        # Layer Normalization for the second sub-layer
+        self.norm2 = keras.layers.LayerNormalization(epsilon=1e-9)
+
+        # Attention mechanism
+        self.attn = AttentionTrain(head_dims=head_dims, num_heads=num_heads, dropout=dropout_rate, input_len = input_len)
+
+        # Dense layer for the first feed-forward sub-layer
+        self.fc1 = keras.layers.Dense(num_heads * head_dims * fc_dim_factor, activation='gelu')
+
+        # Dense layer for the second feed-forward sub-layer
+        self.fc2 = keras.layers.Dense(num_heads * head_dims, activation='gelu')
+
+        # Dropout layers
+        self.dropout1 = keras.layers.Dropout(dropout_rate)
+        self.dropout2 = keras.layers.Dropout(dropout_rate)
+        
+        self._config = {'dropout_rate': dropout_rate}
+
+    def call(self, inputs):
+        x = inputs
+        x = self.attn(x)
+        x = self.dropout1(x)
+        out1 = self.norm1(x + inputs)
+
+        x = out1
+        out1 = self.fc2(self.fc1(out1))
+        out1 = self.dropout2(out1)
+        return self.norm2(out1 + x)

models/embeddings.py

@@ -0,0 +1,17 @@
+import math
+import keras
+import numpy as np
+
+
+class TokenAndPositionEmbedding(keras.layers.Layer):
+    def __init__(self, maxlen, vocab_size, embed_dim):
+        super().__init__()
+        self.token_emb = keras.layers.Embedding(input_dim=vocab_size, output_dim=embed_dim)
+        self.pos_emb = keras.layers.Embedding(input_dim=maxlen, output_dim=embed_dim)
+
+    def call(self, x):
+        maxlen = keras.ops.shape(x)[-1]
+        positions = keras.ops.arange(0, maxlen, 1)
+        positions = self.pos_emb(positions)
+        x = self.token_emb(x)
+        return x + positions

tokenizer/tokenizer.py

@@ -0,0 +1,102 @@
+import sentencepiece as spm
+import os
+import re
+import numpy as np
+
+class SPM_Tokenizer:
+    """
+    A class for tokenizing text data in multiple languages using SentencePiece.
+    
+    Attributes:
+        vocab_model_file (str): The file path to the pre-trained SentencePiece vocabulary model file.
+        vocab_size (int): The size of the vocabulary for training the tokenizer.
+        corpus (str): The file path to the corpus used for training the tokenizer if no pre-trained vocabulary model is provided.
+        model_prefix (str): The prefix for the output files generated during training if no pre-trained vocabulary model is provided.
+        input_size (int): The maximum sequence length for tokenized sequences.
+        model_type (str): The type of SentencePiece model to train, default is "unigram".
+        tokenizer (spm.SentencePieceProcessor): The SentencePiece tokenizer object.
+    
+    Methods:
+        load_file(file_path): Loads and tokenizes text data from a file.
+        load_dataset(list_files): Loads and tokenizes text data from a list of files, yielding input-output pairs for training.
+
+    Examples:
+    ```
+        >>> # Create a new Tokenizer
+        >>> SPM_Tokenizer(vocab_size = 5000, corpus='./stories.txt', input_size=100+1) # Context-width of GPT+1
+        >>> tokenizer = SPM_Tokenizer(vocab_model_file='./tokenizer_.model')
+        Success!
+        >>> tokenizer.tokenizer.encode_as_ids(['Hello World', 'How are you?'])
+        [[3063, 215, 920, 129, 1323], [654, 54, 217, 78]]
+        >>> dataset = tokenizer.load_dataset(['./stories.txt'])
+        >>> for (X, Y) in dataset:
+        >>>     X=np.array(X)[0]
+        >>>     Y=np.array(Y)[0]
+        >>> tokenizer.tokenizer.decode_ids(X.tolist()), tokenizer.tokenizer.decode_ids(Y.tolist())
+        ('The Project Gutenberg EBook of The Thousand and One Nights, Vol. I., by Anonymous This eBook is for the use of anyone anywhere at no cost and with almost no restrictions whatsoever. You may copy it, give it away or re-use it under the',
+         'Project Gutenberg EBook of The Thousand and One Nights, Vol. I., by Anonymous This eBook is for the use of anyone anywhere at no cost and with almost no restrictions whatsoever. You may copy it, give it away or re-use it under the terms')
+        
+    ```
+    """
+    
+    def __init__(self, vocab_model_file=None, vocab_size=5000, corpus=None, model_prefix='tokenizer_', input_size=65, model_type="unigram"):
+        """
+        Initializes the MultiLanguageTokenizer object.
+        
+        Parameters:
+            vocab_model_file (str): The file path to the pre-trained SentencePiece vocabulary model file.
+            vocab_size (int): The size of the vocabulary for training the tokenizer.
+            corpus (str): The file path to the corpus used for training the tokenizer if no pre-trained vocabulary model is provided.
+            model_prefix (str): The prefix for the output files generated during training if no pre-trained vocabulary model is provided.
+            input_size (int): The maximum sequence length for tokenized sequences.
+            model_type (str): The type of SentencePiece model to train, default is "unigram".
+        """
+        self.input_size = input_size
+        if vocab_model_file is not None and os.path.exists(vocab_model_file):
+            self.tokenizer = spm.SentencePieceProcessor()
+            self.tokenizer.load(vocab_model_file)
+        else:
+            if corpus is None:
+                raise Exception('A corpus to train the tokenizer must be provided!')
+
+            self.tokenizer = spm.SentencePieceTrainer.train(input=corpus, model_prefix=model_prefix, vocab_size=vocab_size, model_type=model_type)
+        
+        if self.tokenizer is not None:
+            print('Success!')
+    
+    def load_file(self, file_path):
+        """
+        Loads and tokenizes text data from a file.
+        
+        Parameters:
+            file_path (str): The file path to the text file.
+            
+        Returns:
+            list: A list of tokenized sequences.
+        """
+        with open(file_path, 'r', encoding='utf-8') as file:
+            content = file.readlines()
+            content = " ".join(content).replace('\n', ' ')
+            content = self.tokenizer.encode_as_ids(content)
+            new_content = [content[i:i+self.input_size] for i in range(0, len(content), self.input_size)]
+            num_zeros = self.input_size - len(new_content[-1])
+            padded_list = [0] * num_zeros + new_content[-1]
+            new_content[-1] = padded_list
+        return new_content
+    
+    def load_dataset(self, list_files):
+        """
+        Loads and tokenizes text data from a list of files, yielding input-output pairs for training.
+        
+        Parameters:
+            list_files (list): A list of file paths to text files.
+            
+        Yields:
+            tuple: A tuple containing input and output sequences.
+        """
+        for file in list_files:
+            content = self.load_file(file)
+            X = [line[:-1] for line in content]
+            Y = [line[1:] for line in content]
+            yield X, Y
+

utils/__init__.py

@@ -0,0 +1,2 @@
+from .config import *
+from .utils import *

utils/config.py

@@ -0,0 +1,78 @@
+class _C2G_CONFIG:
+    """
+    This class provides functionalities for managing configurations and modules within the C2G system.
+
+    Methods
+        clean_config(): Resets the configuration dictionary.
+        clean_dir(): Clears the module directory and module list.
+        display_available_modules(): Prints the names of available modules.
+        update_inverse_list(): Updates the inverse dictionary of module names.
+
+    Attributes:
+        _C: A dictionary storing configuration settings.
+        _dir: A dictionary mapping module names to their corresponding objects.
+        _module_list: A list containing names of registered modules.
+
+    Example:
+        ```
+        >>> @register_module('add')
+        >>> def add(a, b):
+        >>>     return a+b
+        >>> add(-10, 12)
+        2
+        >>> cfg.display_available_modules()
+        Available Modules:
+        ADD
+        >>> cfg.clean_dir()
+        >>> cfg.display_available_modules()
+        Available Modules:
+        ```
+    """
+    _C = {}
+    _dir = {}
+    _module_list = []
+
+    def clean_config(self):
+        self._C = {}
+
+    def clean_dir(self):
+        self._dir = {}
+        self._module_list = []
+
+    def display_available_modules(self):
+        print('Available Modules:')
+        for module_name, _ in self._dir.items():
+            print(module_name)
+
+    def update_inverse_list(self):
+        self._inv_dir = {value : key for key, value in self._dir.items()}
+
+
+class register_module:
+    """
+    This class is a decorator used for registering modules within the C2G system.
+
+    Methods
+        __init__(self, name): Initializes the module with a given name.
+        __call__(self, module): Registers the module with the provided name and updates the module list.
+
+    Examples
+        ```
+        >>> @register_module('add')
+        >>> def add(a, b):
+        >>>     return a+b
+        >>> add(-10, 12)
+        2
+        >>> cfg.display_available_modules()
+        Available Modules:
+        ADD
+        ```
+    """
+    def __init__(self, name):
+
+        self.name = name.upper()
+    def __call__(self, module):
+        cfg._dir[self.name] = module
+        cfg.update_inverse_list()
+        cfg._module_list.append(self.name)
+        return cfg._dir[self.name]

utils/utils.py

@@ -0,0 +1,70 @@
+class BASE_C2G:
+    """
+    This is the base class for C2G models.
+
+    Class Methods
+        serialize_c2g(cls, config): Serializes the configuration into a dictionary.
+        deserialize_c2g(self, config): Deserializes the configuration from a dictionary.
+        construct_model(self, configs): Abstract method for constructing a model. This needs to be implemented by subclasses.
+        
+    Examples
+        ```
+        >>> @register_module('test_module')
+        >>> class test_module(BASE_C2G):
+        >>>     def __init__(self, a, b):
+        >>>         self._config = {'a':a, 'b':b}
+        >>>         self.a = a
+        >>>         self.b = b
+        >>>     def __call__(self):
+        >>>         return self.a+self.b
+
+        >>> sub_module = test_module(1, 2)
+        >>> sub_module()
+        3
+
+        >>> @register_module('master_module')
+        >>> class master_module(BASE_C2G):
+        >>>     def __init__(self, sub_module):
+        >>>         self._config = {'sub_module': sub_module}
+        >>>         self.sub_module = sub_module
+
+        >>>     def __call__(self):
+        >>>         print(self.sub_module())
+
+        >>> master_mod = master_module(sub_module)
+        >>> master_mod()
+        3
+
+        >>> save_master_module = BASE_C2G()
+        >>> save_master_module.serialize_c2g(master_mod._config)
+        {'BASE_C2G': {'SUB_MODULE': {'TEST_MODULE': {'a': 1, 'b': 2}}}}
+        >>> cfg._module_list
+        ['TEST_MODULE', 'MASTER_MODULE']
+        ```
+
+    """
+    @classmethod
+    def serialize_c2g(cls, config):
+        cls_config = {}
+        for keyword, value in config.items():
+            if value.__class__.__name__.upper() not in cfg._module_list:
+            #if value.__name__.upper() not in cfg._module_list:
+                cls_config[keyword] = value
+            else:
+                cls_config[keyword.upper()] = value.serialize_c2g(value._config)
+        cfg._C = {cls.__name__.upper(): cls_config}
+        return cfg._C
+
+    def has_sub_dicts(self, config):
+        for key, val in config.items():
+            if isinstance(val, dict):
+                return False
+        return True
+
+    def deserialize_c2g(self, config):
+        for model, params in config.items():
+            pass
+        return cfg._dir[model](**params)
+
+    def construct_model(self, configs):
+        raise NotImplementedError('`construct_model` method is not implemented. To implement it, define it on the subclasses of BASE_C2G Models.')