Upload 5 files

Decoder.ipynb

@@ -0,0 +1,165 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import math\n",
+    "import torch \n",
+    "import torch.nn as nn\n",
+    "\n",
+    "class InputEmbeddingsLayer(nn.Module):\n",
+    "    def __init__(self, d_model: int, vocab_size: int) -> None:\n",
+    "        super().__init__()\n",
+    "        self.d_model = d_model\n",
+    "        self.vocab_size = vocab_size\n",
+    "        self.embedding = nn.Embedding(vocab_size, d_model)\n",
+    "    def forward(self, x):\n",
+    "        return self.embedding(x) * math.sqrt(self.d_model)\n",
+    "\n",
+    "class PositionalEncodingLayer(nn.Module):\n",
+    "    def __init__(self, d_model: int, sequence_length: int, dropout: float) -> None:\n",
+    "        super().__init__()\n",
+    "        self.d_model = d_model\n",
+    "        self.sequence_length = sequence_length\n",
+    "        self.dropout = nn.Dropout(dropout)\n",
+    "\n",
+    "        PE = torch.zeros(sequence_length, d_model)\n",
+    "        Position = torch.arange(0, sequence_length, dtype=torch.float).unsqueeze(1)\n",
+    "        deviation_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n",
+    "        \n",
+    "        PE[:, 0::2] = torch.sin(Position * deviation_term)\n",
+    "        PE[:, 1::2] = torch.cos(Position * deviation_term)\n",
+    "        PE = PE.unsqueeze(0)\n",
+    "        self.register_buffer(\"PE\", PE)\n",
+    "    def forward(self, x):\n",
+    "        x = x + (self.PE[:, :x.shape[1], :]).requires_grad_(False)\n",
+    "        return self.dropout(x)\n",
+    "\n",
+    "class NormalizationLayer(nn.Module):\n",
+    "    def __init__(self, Epslone: float = 10**-6) -> None:\n",
+    "        super().__init__()\n",
+    "        self.Epslone = Epslone\n",
+    "        self.Alpha = nn.Parameter(torch.ones(1))\n",
+    "        self.Bias = nn.Parameter(torch.ones(1))\n",
+    "    def forward(self, x):\n",
+    "        mean = x.mean(dim = -1, keepdim = True)\n",
+    "        std = x.std(dim = -1, keepdim = True)\n",
+    "        return self.Alpha * (x - mean) / (std + self.Epslone) + self.Bias\n",
+    "\n",
+    "class FeedForwardBlock(nn.Module):\n",
+    "    def __init__(self, d_model: int, d_ff: int, dropout: float) -> None:\n",
+    "        super().__init__()\n",
+    "        self.Linear_1 = nn.Linear(d_model, d_ff)\n",
+    "        self.dropout = nn.Dropout(dropout)\n",
+    "        self.Linear_2 = nn.Linear(d_ff, d_model)\n",
+    "    def forward(self, x):\n",
+    "        return self.Linear_2(self.dropout(torch.relu(self.Linear_1(x))))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class MultiHeadAttentionBlock(nn.Module):\n",
+    "    def __init__(self, d_model: int, heads: int, dropout: float) -> None:\n",
+    "        super().__init__()\n",
+    "        self.d_model = d_model\n",
+    "        self.heads = heads \n",
+    "        assert d_model % heads == 0, \"d_model is not divisable by heads\"\n",
+    "\n",
+    "        self.d_k = d_model // heads\n",
+    "\n",
+    "        self.W_Q = nn.Linear(d_model, d_model)\n",
+    "        self.W_K = nn.Linear(d_model, d_model)\n",
+    "        self.W_V = nn.Linear(d_model, d_model)\n",
+    "\n",
+    "        self.W_O = nn.Linear(d_model, d_model)\n",
+    "        self.dropout = nn.Dropout(dropout)\n",
+    "    \n",
+    "    @staticmethod\n",
+    "    def Attention(Query, Key, Value, mask, dropout: nn.Module):\n",
+    "        d_k = Query.shape[-1]\n",
+    "\n",
+    "        self_attention_score = (Query @ Key.transpose(-2,-1)) / math.sqrt(d_k)\n",
+    "        if mask is not None:\n",
+    "            self_attention_score.masked_fill_(mask == 0, -1e9)\n",
+    "        self_attention_score = self_attention_score.softmax(dim = -1)\n",
+    "\n",
+    "        if dropout is not None:\n",
+    "            self_attention_score = dropout(self_attention_score)\n",
+    "        return self_attention_score @ Value\n",
+    "    def forward(self, query, key, value, mask):\n",
+    "        Query = self.W_Q(query)\n",
+    "        Key = self.W_K(key)\n",
+    "        Value = self.W_V(value)\n",
+    "\n",
+    "        Query = Query.view(Query.shape[0], Query.shape[1], self.heads, self.d_k).transpose(1,2)\n",
+    "        Key = Key.view(Key.shape[0], Key.shape[1], self.heads, self.d_k).transpose(1,2)\n",
+    "        Value = Value.view(Value.shape[0], Value.shape[1], self.heads, self.d_k).transpose(1,2)\n",
+    "\n",
+    "        x, self.self_attention_score = MultiHeadAttentionBlock.Attention(Query, Key, Value, mask, self.dropout)\n",
+    "        x = x.transpose(1,2).contiguous().view(x.shape[0], -1, self.heads * self.d_k)\n",
+    "        return self.W_O(x)\n",
+    "\n",
+    "class ResidualConnection(nn.Module):\n",
+    "    def __init__(self, dropout: float) -> None:\n",
+    "        super().__init__()\n",
+    "        self.dropout = nn.Dropout(dropout)\n",
+    "        self.normalization = NormalizationLayer()\n",
+    "    def forward(self, x, subLayer):\n",
+    "        return x + self.dropout(subLayer(self.normalization(x)))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Building the decoder block \n",
+    "class DecoderBlock(nn.Module):\n",
+    "    def __init__(self, decoder_self_attention_block: MultiHeadAttentionBlock, decoder_cross_attention_block: MultiHeadAttentionBlock, decoder_feed_forward_block: FeedForwardBlock, dropout: float) -> None:\n",
+    "        super().__init__()\n",
+    "        self.decoder_self_attention_block = decoder_self_attention_block\n",
+    "        self.decoder_cross_attention_block = decoder_cross_attention_block\n",
+    "        self.decoder_feed_forward_block = decoder_feed_forward_block\n",
+    "        self.residual_connection = nn.ModuleList([ResidualConnection(dropout) for _ in range(3)])\n",
+    "    def forward(self, x, Encoder_output, source_mask, target_mask):\n",
+    "        x = self.residual_connection[0](x, lambda x: self.decoder_self_attention_block(x, x, x, source_mask))\n",
+    "        x = self.residual_connection[1](x, lambda x: self.decoder_cross_attention_block(x, Encoder_output, Encoder_output, target_mask))\n",
+    "        x = self.residual_connection[2](x, self.decoder_feed_forward_block)\n",
+    "        return x\n",
+    "\n",
+    "class Decoder(nn.Module):\n",
+    "    def __init__(self, Layers: nn.ModuleList) -> None:\n",
+    "        super().__init__()\n",
+    "        self.Layers = Layers\n",
+    "        self.normalization = NormalizationLayer()\n",
+    "    def forward(self, x, Encoder_output, source_mask, target_mask):\n",
+    "        for layer in self.Layers:\n",
+    "            x = layer(x, Encoder_output, source_mask, target_mask)\n",
+    "        return self.normalization(x)\n",
+    "\n",
+    "class LinearLayer(nn.Module):\n",
+    "    def __init__(self, d_model: int, vocab_size: int) -> None:\n",
+    "        super().__init__()\n",
+    "        self.Linear = nn.Linear(d_model, vocab_size)\n",
+    "    def forward(self, x):\n",
+    "        return self.Linear(x)"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

Encoder.ipynb

@@ -0,0 +1,156 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import math\n",
+    "import torch \n",
+    "import torch.nn as nn\n",
+    "\n",
+    "class InputEmbeddingsLayer(nn.Module):\n",
+    "    def __init__(self, d_model: int, vocab_size: int) -> None:\n",
+    "        super().__init__()\n",
+    "        self.d_model = d_model\n",
+    "        self.vocab_size = vocab_size\n",
+    "        self.embedding = nn.Embedding(vocab_size, d_model)\n",
+    "    def forward(self, x):\n",
+    "        return self.embedding(x) * math.sqrt(self.d_model)\n",
+    "\n",
+    "class PositionalEncodingLayer(nn.Module):\n",
+    "    def __init__(self, d_model: int, sequence_length: int, dropout: float) -> None:\n",
+    "        super().__init__()\n",
+    "        self.d_model = d_model\n",
+    "        self.sequence_length = sequence_length\n",
+    "        self.dropout = nn.Dropout(dropout)\n",
+    "\n",
+    "        PE = torch.zeros(sequence_length, d_model)\n",
+    "        Position = torch.arange(0, sequence_length, dtype=torch.float).unsqueeze(1)\n",
+    "        deviation_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n",
+    "        \n",
+    "        PE[:, 0::2] = torch.sin(Position * deviation_term)\n",
+    "        PE[:, 1::2] = torch.cos(Position * deviation_term)\n",
+    "        PE = PE.unsqueeze(0)\n",
+    "        self.register_buffer(\"PE\", PE)\n",
+    "    def forward(self, x):\n",
+    "        x = x + (self.PE[:, :x.shape[1], :]).requires_grad_(False)\n",
+    "        return self.dropout(x)\n",
+    "\n",
+    "class NormalizationLayer(nn.Module):\n",
+    "    def __init__(self, Epslone: float = 10**-6) -> None:\n",
+    "        super().__init__()\n",
+    "        self.Epslone = Epslone\n",
+    "        self.Alpha = nn.Parameter(torch.ones(1))\n",
+    "        self.Bias = nn.Parameter(torch.ones(1))\n",
+    "    def forward(self, x):\n",
+    "        mean = x.mean(dim = -1, keepdim = True)\n",
+    "        std = x.std(dim = -1, keepdim = True)\n",
+    "        return self.Alpha * (x - mean) / (std + self.Epslone) + self.Bias\n",
+    "\n",
+    "class FeedForwardBlock(nn.Module):\n",
+    "    def __init__(self, d_model: int, d_ff: int, dropout: float) -> None:\n",
+    "        super().__init__()\n",
+    "        self.Linear_1 = nn.Linear(d_model, d_ff)\n",
+    "        self.dropout = nn.Dropout(dropout)\n",
+    "        self.Linear_2 = nn.Linear(d_ff, d_model)\n",
+    "    def forward(self, x):\n",
+    "        return self.Linear_2(self.dropout(torch.relu(self.Linear_1(x))))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class MultiHeadAttentionBlock(nn.Module):\n",
+    "    def __init__(self, d_model: int, heads: int, dropout: float) -> None:\n",
+    "        super().__init__()\n",
+    "        self.d_model = d_model\n",
+    "        self.heads = heads \n",
+    "        assert d_model % heads == 0, \"d_model is not divisable by heads\"\n",
+    "\n",
+    "        self.d_k = d_model // heads\n",
+    "\n",
+    "        self.W_Q = nn.Linear(d_model, d_model)\n",
+    "        self.W_K = nn.Linear(d_model, d_model)\n",
+    "        self.W_V = nn.Linear(d_model, d_model)\n",
+    "\n",
+    "        self.W_O = nn.Linear(d_model, d_model)\n",
+    "        self.dropout = nn.Dropout(dropout)\n",
+    "    \n",
+    "    @staticmethod\n",
+    "    def Attention(Query, Key, Value, mask, dropout: nn.Module):\n",
+    "        d_k = Query.shape[-1]\n",
+    "\n",
+    "        self_attention_score = (Query @ Key.transpose(-2,-1)) / math.sqrt(d_k)\n",
+    "        if mask is not None:\n",
+    "            self_attention_score.masked_fill_(mask == 0, -1e9)\n",
+    "        self_attention_score = self_attention_score.softmax(dim = -1)\n",
+    "\n",
+    "        if dropout is not None:\n",
+    "            self_attention_score = dropout(self_attention_score)\n",
+    "        return self_attention_score @ Value\n",
+    "    def forward(self, query, key, value, mask):\n",
+    "        Query = self.W_Q(query)\n",
+    "        Key = self.W_K(key)\n",
+    "        Value = self.W_V(value)\n",
+    "\n",
+    "        Query = Query.view(Query.shape[0], Query.shape[1], self.heads, self.d_k).transpose(1,2)\n",
+    "        Key = Key.view(Key.shape[0], Key.shape[1], self.heads, self.d_k).transpose(1,2)\n",
+    "        Value = Value.view(Value.shape[0], Value.shape[1], self.heads, self.d_k).transpose(1,2)\n",
+    "\n",
+    "        x, self.self_attention_score = MultiHeadAttentionBlock.Attention(Query, Key, Value, mask, self.dropout)\n",
+    "        x = x.transpose(1,2).contiguous().view(x.shape[0], -1, self.heads * self.d_k)\n",
+    "        return self.W_O(x)\n",
+    "\n",
+    "class ResidualConnection(nn.Module):\n",
+    "    def __init__(self, dropout: float) -> None:\n",
+    "        super().__init__()\n",
+    "        self.dropout = nn.Dropout(dropout)\n",
+    "        self.normalization = NormalizationLayer()\n",
+    "    def forward(self, x, subLayer):\n",
+    "        return x + self.dropout(subLayer(self.normalization(x)))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Building the encoder block \n",
+    "class EncoderBlock(nn.Module):\n",
+    "    def __init__(self, encoder_self_attention_block: MultiHeadAttentionBlock, encoder_feed_forward_block: FeedForwardBlock, dropout: float) -> None:\n",
+    "        super().__init__()\n",
+    "        self.encoder_self_attention_block = encoder_self_attention_block\n",
+    "        self.encoder_feed_forward_block = encoder_feed_forward_block\n",
+    "        self.residual_connection = nn.ModuleList([ResidualConnection(dropout) for _ in range(2)])\n",
+    "    def forward(self, x, source_mask):\n",
+    "        x = self.residual_connection[0](x, lambda x: self.encoder_self_attention_block(x, x, x, source_mask))\n",
+    "        x = self.residual_connection[1](x, self.encoder_feed_forward_block)\n",
+    "        return x\n",
+    "\n",
+    "class Encoder(nn.Module):\n",
+    "    def __init__(self, Layers: nn.ModuleList) -> None:\n",
+    "        super().__init__()\n",
+    "        self.Layers = Layers\n",
+    "        self.normalization = NormalizationLayer()\n",
+    "    def forward(self, x, source_mask):\n",
+    "        for layer in self.Layers:\n",
+    "            x = layer(x, source_mask)\n",
+    "        return self.normalization(x)"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

TransformerBlock.ipynb

@@ -0,0 +1,143 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import import_ipynb\n",
+    "from Encoder import Encoder, EncoderBlock, MultiHeadAttentionBlock, FeedForwardBlock, InputEmbeddingsLayer, PositionalEncodingLayer\n",
+    "from Decoder import Decoder, DecoderBlock, MultiHeadAttentionBlock, FeedForwardBlock, InputEmbeddingsLayer, PositionalEncodingLayer\n",
+    "\n",
+    "import torch\n",
+    "import torch.nn as nn \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class LinearLayer(nn.Module):\n",
+    "\n",
+    "    def __init__(self, d_model: int, vocab_size: int) -> None:\n",
+    "        super().__init__()\n",
+    "        self.Linear = nn.Linear(d_model, vocab_size)\n",
+    "\n",
+    "    def forward(self, x):\n",
+    "        return self.Linear(x)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class TransformerBlock(nn.Module):\n",
+    "\n",
+    "    def __init__(self, encoder: Encoder, decoder: Decoder, source_embedding: InputEmbeddingsLayer, target_embedding: InputEmbeddingsLayer, source_position: PositionalEncodingLayer, target_position: PositionalEncodingLayer, Linear: LinearLayer) -> None:\n",
+    "        super().__init__()\n",
+    "        self.encoder = encoder \n",
+    "        self.decoder = decoder \n",
+    "        self.source_embedding = source_embedding\n",
+    "        self.target_embedding = target_embedding\n",
+    "        self.source_position = source_position\n",
+    "        self.target_position = target_position\n",
+    "        self.Linear = Linear\n",
+    "\n",
+    "    def encode(self, source_language, source_mask):\n",
+    "        source_language = self.source_embedding(source_language)\n",
+    "        source_language = self.source_position(source_language)\n",
+    "        return self.encoder(source_language, source_mask)\n",
+    "\n",
+    "    def decode(self, Encoder_output, source_mask, target_language, target_mask):\n",
+    "        target_language = self.target_embedding(target_language)\n",
+    "        target_language = self.target_position(target_language)\n",
+    "        return self.decoder(target_language, Encoder_output, source_mask, target_mask)\n",
+    "\n",
+    "    def linear(self, x):\n",
+    "        return self.Linear(x)\n",
+    "    \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def Transformer_Model(source_vocab_size: int, target_vocab_size: int, source_sequence_length: int, target_sequence_length: int, d_model: int = 512, Layers: int = 6, heads: int = 8, dropout: float = 0.1, d_ff: int = 2048)->TransformerBlock:\n",
+    "\n",
+    "    source_embedding = InputEmbeddingsLayer(d_model, source_vocab_size)\n",
+    "    target_embedding = InputEmbeddingsLayer(d_model, target_vocab_size)\n",
+    "\n",
+    "    source_position = PositionalEncodingLayer(d_model, source_sequence_length, dropout)\n",
+    "    target_position = PositionalEncodingLayer(d_model, target_sequence_length, dropout)\n",
+    "\n",
+    "    EncoderBlocks = []\n",
+    "    for _ in range(Layers):\n",
+    "        encoder_self_attention_block = MultiHeadAttentionBlock(d_model, heads, dropout)\n",
+    "        encoder_feed_forward_block = FeedForwardBlock(d_model, d_ff, dropout)\n",
+    "        encoder_block = EncoderBlock(encoder_self_attention_block, encoder_feed_forward_block, dropout)\n",
+    "        EncoderBlocks.append(encoder_block)\n",
+    "\n",
+    "    DecoderBlocks = []\n",
+    "    for _ in range(Layers):\n",
+    "        decoder_self_attention_block = MultiHeadAttentionBlock(d_model, heads, dropout)\n",
+    "        decoder_cross_attention_block = MultiHeadAttentionBlock(d_model, heads, dropout)\n",
+    "        decoder_feed_forward_block = FeedForwardBlock(d_model, d_ff, dropout)\n",
+    "        decoder_block = DecoderBlock(decoder_self_attention_block, decoder_cross_attention_block, decoder_feed_forward_block, dropout)\n",
+    "        DecoderBlocks.append(decoder_block)\n",
+    "\n",
+    "    encoder = Encoder(nn.ModuleList(EncoderBlocks))\n",
+    "    decoder = Decoder(nn.ModuleList(DecoderBlocks))\n",
+    "\n",
+    "    linear = LinearLayer(d_model, target_vocab_size)\n",
+    "\n",
+    "    Transformer = TransformerBlock(encoder, decoder, source_embedding, target_embedding, source_position, target_position, linear)\n",
+    "    \n",
+    "    for t in Transformer.parameters():\n",
+    "        if t.dim() > 1:\n",
+    "            nn.init.xavier_uniform(t)\n",
+    "\n",
+    "    return Transformer\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "5f594f1fbc6ec12c92a2efee092a20dcfd0697dc036fc348ba81f2fc261c5e29"
+  },
+  "kernelspec": {
+   "display_name": "Python 3.11.5 64-bit",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.5"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

transformer.py

@@ -0,0 +1,241 @@
+from dataclasses import dataclass
+import math
+import torch
+import torch.nn as nn 
+
+@dataclass
+class Args:
+    source_vocab_size: int
+    target_vocab_size: int 
+    source_sequence_length: int 
+    target_sequence_length: int 
+    d_model: int = 512
+    Layers: int = 6 
+    heads: int = 8 
+    dropout: float = 0.1 
+    d_ff: int = 2048 
+
+class InputEmbeddingLayer(nn.Module):
+    def __init__(self, d_model: int, vocab_size: int) -> None:
+        super().__init__()
+        self.d_model = d_model
+        self.vocab_size = vocab_size
+        self.embedding = nn.Embedding(vocab_size, d_model)
+
+    def forward(self, x):
+        return self.embedding(x) * math.sqrt(self.d_model)
+
+class PositionalEncodingLayer(nn.Module):
+    def __init__(self, d_model: int, sequence_length: int, dropout: float) -> None:
+        super().__init__()
+        self.d_model = d_model
+        self.sequence_length = sequence_length
+        self.dropout = nn.Dropout(dropout)
+
+        PE = torch.zeros(sequence_length, d_model)
+        Position = torch.arange(0, sequence_length, dtype=torch.float).unsqueeze(1)
+        deviation_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
+
+        PE[:, 0::2] = torch.sin(Position * deviation_term)
+        PE[:, 1::2] = torch.cos(Position * deviation_term)
+        PE = PE.unsqueeze(0)
+        self.register_buffer('PE', PE)
+    
+    def forward(self, x):
+        x = x + (self.PE[:, :x.shape[1], :]).requires_grad(False)
+        return self.dropout(x)
+
+class NormalizationLayer(nn.Module):
+    def __init__(self, Epsilon: float = 10**-4) -> None:
+        super().__init__()
+        self.Epsilon = Epsilon
+        self.Alpha = nn.Parameter(torch.ones(1))
+        self.Bias = nn.Parameter(torch.ones(1))
+    
+    def forward(self, x):
+        mean = x.mean(dim = -1, keepdim = True)
+        std = x.std(dim = -1, keepdim = True)
+        return self.Alpha * (x - mean) /  (std + self.Epsilon) + self.Bias 
+
+class FeedForwardBlock(nn.Module):
+    def __init__(self, d_model: int, d_ff: int, dropout: float) -> None:
+        super().__init__()
+        self.Linear_1 = nn.Linear(d_model, d_ff)
+        self.dropout = nn.Dropout(dropout)
+        self.Linear_2 = nn.Linear(d_ff, d_model)
+    
+    def forward(self, x):
+        return self.Linear_2(self.dropout(torch.relu(self.Linear_1(x))))
+
+class MultiHeadAttentionBlock(nn.Module):
+    def __init__(self, d_model: int, heads: int, dropout: float) -> None:
+        super().__init__()
+        self.d_model = d_model
+        self.heads = heads 
+        assert d_model % heads == 0, "d_model is not divisable by heads"
+
+        self.d_k = d_model // heads 
+
+        self.W_Q = nn.Linear(d_model, d_model)
+        self.W_K = nn.Linear(d_model, d_model)
+        self.W_V = nn.Linear(d_model, d_model)
+        self.W_O = nn.Linear(d_model, d_model)
+
+        self.dropout = nn.Dropout(dropout)
+    
+    @staticmethod
+    def Attention(Query, Key, Value, mask, dropout):
+        d_k = Query.shape[-1]
+        self_attention_scores = (Query @ Key.traspose(-2, -1)) / math.sqrt(d_k)
+
+        if mask is not None:
+            self_attention_scores.masked_fill(mask == 0, -1e9)
+        self_attention_scores = self_attention_scores.Softmax(dim = -1)
+
+        if dropout is not None:
+            self_attention_scores = dropout(self_attention_scores)
+        return self_attention_scores @ Value
+
+    def forward(self, query, key, value, mask):
+        Query = self.W_Q(query)
+        Key = self.W_K(key)
+        Value = self.W_V(value)
+
+        Query = Query.view(Query.shape[0], Query.shape[1], self.heads, self.d_k).transpose(1,2)
+        Key = Key.view(Key.shape[0], Key.shape[1], self.heads, self.d_k).transpose(1,2)
+        Value = Value.view(Value.shape[0], Value.shape[1], self.heads, self.d_k).transpose(1,2)
+
+        x, self.self_attention_scores = MultiHeadAttentionBlock.Attention(Query, Key, Value, mask, self.dropout)
+        x = x.transpose().contiguous().view(x.shape[0], -1, self.heads * self.d_k)
+        return self.W_O(x)
+
+class ResidualConnection(nn.Module):
+    def __init__(self, dropout: float) -> None:
+        super().__init__()
+        self.dropout = nn.Dropout(dropout)
+        self.normalization_layer = NormalizationLayer()
+    
+    def forward(self, x, subLayer):
+        return self.dropout(subLayer(self.normalization_layer))
+
+class EncoderBlock(nn.Module):
+    def __init__(self, self_attetion_block: MultiHeadAttentionBlock, feed_forward_block: FeedForwardBlock, dropout: float) -> None:
+        super().__init__()
+        self.self_attention_block = self_attetion_block
+        self.feed_forward_block = feed_forward_block
+        self.residual_connection = nn.ModuleList([ResidualConnection(dropout) for _ in range(2)])
+
+    def forward(self, x, source_mask):
+        x = self.residual_connection[0](x, lambda x: self.self_attention_block(x, x, x, source_mask))
+        x = self.residual_connection[1](x, self.feed_forward_block)
+        return x 
+
+class Encoder(nn.Module):
+    def __init__(self, Layers: nn.ModuleList) -> None:
+        super().__init__()
+        self.Layers = Layers
+        self.normalization_layer = NormalizationLayer()
+
+    def forward(self, x, source_mask):
+        for layer in self.Layers:
+            x = layer(x, source_mask)
+        return self.normalization_layer(x)
+
+class DecoderBlock(nn.Module):
+    def __init__(self, masked_self_attention_block: MultiHeadAttentionBlock, self_attention_block: MultiHeadAttentionBlock, feedforwardblock: FeedForwardBlock, dropout: float) -> None:
+        super().__init__()
+        self.masked_self_attention_block = masked_self_attention_block
+        self.self_attention_block = self_attention_block
+        self.feedforwardblock = feedforwardblock
+        self.residual_connection = nn.ModuleList([ResidualConnection(dropout) for _ in range(3)])
+    
+    def forward(self, x, Encoder_output, source_mask, target_mask):
+        x = self.residual_connection[0](x, lambda x: self.masked_self_attention_block(x, x, x, source_mask))
+        x = self.residual_connection[1](x, lambda x: self.self_attention_block(x, Encoder_output, Encoder_output, target_mask))
+        x = self.residual_connection[1](x, self.feedforwardblock)
+        return x
+
+class Decoder(nn.Module):
+    def __init__(self, Layers: nn.ModuleList) -> None:
+        super().__init__()
+        self.Layers = Layers 
+        self.normalization_layer = NormalizationLayer()
+    
+    def forward(self, x, Encoder_output, source_mask, target_mask):
+        for layer in self.Layers:
+            x = layer(x, Encoder_output, source_mask, target_mask)
+        return self.normalization_layer(x)
+
+class LinearLayer(nn.Module):
+    def __init__(self, d_model: int, vocab_size: int) -> None:
+        super().__init__()
+        self.Linear = nn.Linear(d_model, vocab_size)
+    
+    def forward(self, x):
+        return self.Linear(x)
+
+class TransformerBlock(nn.Module):
+    def __init__(self, encoder: Encoder, 
+                       decoder: Decoder, 
+                       source_embedding: InputEmbeddingLayer, 
+                       target_embedding: InputEmbeddingLayer, 
+                       source_position: PositionalEncodingLayer, 
+                       target_position: PositionalEncodingLayer, 
+                       Linear: LinearLayer) -> None:
+        super().__init__()
+        self.encoder = encoder
+        self.decoder = decoder 
+        self.source_embedding = source_embedding
+        self.target_embedding = target_embedding
+        self.source_position = source_position
+        self.target_position = target_position
+        self.Linear = Linear
+    
+    def encode(self, source_language, source_mask):
+        source_language = self.source_embedding(source_language)
+        source_language = self.source_position(source_language)
+        return self.encoder(source_language, source_mask)
+    
+    def decode(self, Encoder_output, source_mask, target_language, target_mask):
+        target_language = self.target_embedding(target_language)
+        target_language = self.target_position(target_language)
+        return self.decoder(target_language, Encoder_output, source_mask, target_mask)
+    
+    def linear(self, x):
+        return self.Linear(x)
+
+
+def Transformer_model(Args: Args)->TransformerBlock:
+
+    source_embedding = InputEmbeddingLayer(Args.d_model, Args.source_vocab_size)
+    source_position = PositionalEncodingLayer(Args.d_model, Args.source_sequence_length, Args.dropout)
+
+    target_embedding = InputEmbeddingLayer(Args.d_model, Args.target_vocab_size)
+    target_position = PositionalEncodingLayer(Args.d_model, Args.target_sequence_length, Args.dropout)
+
+    Encoder_Blocks = []
+    for _ in range(Args.Layers):
+        encoder_self_attention_block = MultiHeadAttentionBlock(Args.d_model, Args.heads, Args.dropout)
+        encoder_feed_forward_block = FeedForwardBlock(Args.d_model, Args.d_ff, Args.dropout)
+        encoder_block = EncoderBlock(encoder_self_attention_block, encoder_feed_forward_block, Args.dropout)
+        Encoder_Blocks.append(encoder_block)
+
+    Decoder_Blocks = []
+    for _ in range(Args.Layers):
+        decoder_self_attention_block = MultiHeadAttentionBlock(Args.d_model, Args.heads, Args.dropout)
+        decoder_cross_attention_block = MultiHeadAttentionBlock(Args.d_model, Args.heads, Args.dropout)
+        decoder_feed_forward_block = FeedForwardBlock(Args.d_model, Args.d_ff, Args.dropout)
+        decoder_block = DecoderBlock(decoder_self_attention_block, decoder_cross_attention_block, decoder_feed_forward_block, Args.dropout)
+        Decoder_Blocks.append(decoder_block)
+
+    encoder = Encoder(nn.ModuleList(Encoder_Blocks))
+    decoder = Decoder(nn.ModuleList(Decoder_Blocks))
+
+    linear = LinearLayer(Args.d_model, Args.target_vocab_size)
+
+    Transformer = TransformerBlock(encoder, decoder, source_embedding, target_embedding, source_position, target_position, linear)
+
+    for t in Transformer.parameters():
+        if t.dim() > 1:
+            nn.init.xavier_uniform(t)
+    return Transformer