{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3UeycYCyxDfE"
      },
      "source": [
        "# TRANSLATOR"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8BfUjVxBcz5N"
      },
      "source": [
        "## instalation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WXqM38xBRHu2"
      },
      "outputs": [],
      "source": [
        "%%time\n",
        "!pip install tensorflow-text\n",
        "!pip install datasets\n",
        "!pip install -q tensorflow_datasets\n",
        "!pip install pydot\n",
        "!cd /content\n",
        "!clear"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ukvs1XfMG7aG"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "import tensorflow_text as tf_text\n",
        "import tensorflow_datasets as tfds\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import requests\n",
        "import functools\n",
        "import collections\n",
        "import os\n",
        "import pathlib\n",
        "import re\n",
        "import string\n",
        "import tempfile\n",
        "import time\n",
        "import matplotlib.pyplot as plt\n",
        "import os\n",
        "import re\n",
        "import shutil\n",
        "import string\n",
        "import tensorflow as tf\n",
        "\n",
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras import losses\n",
        "import pydot"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7V7igFwpc6Hs"
      },
      "source": [
        "## dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZaMtoUtAREzs"
      },
      "outputs": [],
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "dataset = load_dataset(\"Helsinki-NLP/opus_books\", \"en-fr\")\n",
        "data = dataset[\"train\"]\n",
        "\n",
        "french_sentences = [example[\"fr\"] for example in data[\"translation\"][:127085]]\n",
        "english_sentences = [example[\"en\"] for example in data[\"translation\"][:127085]]\n",
        "dataset = tf.data.Dataset.from_tensor_slices((french_sentences, english_sentences))\n",
        "\n",
        "french_sentences_decoded = []\n",
        "english_sentences_decoded = []\n",
        "\n",
        "for french_sentence, english_sentence in dataset.take(127085):\n",
        "    french_sentences_decoded.append(\"b '\"+french_sentence.numpy().decode('utf-8'))\n",
        "    english_sentences_decoded.append(\"b '\"+english_sentence.numpy().decode('utf-8'))\n",
        "\n",
        "print(\"Nombre de phrases en français :\", len(french_sentences_decoded))\n",
        "print(\"Nombre de phrases en anglais :\", len(english_sentences_decoded))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2tb7H5uFQoBA"
      },
      "outputs": [],
      "source": [
        "train_fr = french_sentences\n",
        "train_en = english_sentences"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NzS8h0budHWv"
      },
      "source": [
        "## vocab"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "72uMXsQhFIx8"
      },
      "outputs": [],
      "source": [
        "from tensorflow_text.tools.wordpiece_vocab import bert_vocab_from_dataset as bert_vocab\n",
        "\n",
        "bert_tokenizer_params = dict(lower_case=True)\n",
        "reserved_tokens = [\"[PAD]\", \"[UNK]\", \"[START]\", \"[END]\"]\n",
        "\n",
        "bert_vocab_args = {\n",
        "    'vocab_size': 8000,\n",
        "    'reserved_tokens': reserved_tokens,\n",
        "    'bert_tokenizer_params': bert_tokenizer_params,\n",
        "    'learn_params': {},\n",
        "}\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "q6f3mrA-DK0n"
      },
      "outputs": [],
      "source": [
        "%%time\n",
        "en_vocab = bert_vocab.bert_vocab_from_dataset(\n",
        "    tf.data.Dataset.from_tensor_slices(english_sentences).batch(1000).prefetch(2),\n",
        "    **bert_vocab_args\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EJZzz1x5YY0x"
      },
      "outputs": [],
      "source": [
        "%%time\n",
        "fr_vocab = bert_vocab.bert_vocab_from_dataset(\n",
        "    tf.data.Dataset.from_tensor_slices(french_sentences).batch(1000).prefetch(2),\n",
        "    **bert_vocab_args\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1hmXuHHNcBHg"
      },
      "outputs": [],
      "source": [
        "def write_vocab_file(filepath, vocab):\n",
        "  with open(filepath, 'w') as f:\n",
        "    for token in vocab:\n",
        "      print(token, file=f)\n",
        "write_vocab_file('en_vocab.txt', en_vocab)\n",
        "write_vocab_file('fr_vocab.txt', fr_vocab)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kLTf_mEvfNR9"
      },
      "source": [
        "#` TOKENIZER `\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jGshnQ2idy8I"
      },
      "outputs": [],
      "source": [
        "fr_tokenizer = tf_text.BertTokenizer('fr_vocab.txt', **bert_tokenizer_params)\n",
        "en_tokenizer = tf_text.BertTokenizer('en_vocab.txt', **bert_tokenizer_params)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bbQyYKhHkkDe"
      },
      "outputs": [],
      "source": [
        "# Tokenize the examples -> (batch, word, word-piece)\n",
        "en_tokenizere = en_tokenizer.tokenize(\"hello how are you Vadim\")\n",
        "# Merge the word and word-piece axes -> (batch, tokens)\n",
        "en_tokenizere= en_tokenizere.merge_dims(-2,-1)\n",
        "\n",
        "for ex in en_tokenizere.to_list():\n",
        "  print(ex)\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "words = en_tokenizer.detokenize(token_batch)\n",
        "tf.strings.reduce_join(words, separator=' ', axis=-1)"
      ],
      "metadata": {
        "id": "k0m1461Gwy3e"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## model"
      ],
      "metadata": {
        "id": "BjoPdwoxBWw2"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7OqoDOybbwi6"
      },
      "outputs": [],
      "source": [
        "max_length = 200\n",
        "\n",
        "fr_sequences = [fr_tokenizer.tokenize(french_sentence.numpy().decode('utf-8')).merge_dims(-2,-1)\n",
        "                for french_sentence, _ in dataset.take(127085)]\n",
        "fr_ragged = tf.ragged.stack(fr_sequences)\n",
        "fr_padded = fr_ragged.to_tensor(default_value=0, shape=[None, None, max_length])\n",
        "\n",
        "fr_sequencesdeocde = [fr_tokenizer.tokenize(\"[START]\"+french_sentence.numpy().decode('utf-8')+\"[END]\").merge_dims(-2,-1)\n",
        "                for french_sentence, _ in dataset.take(127085)]\n",
        "fr_raggeddecode = tf.ragged.stack(fr_sequences)\n",
        "fr_paddeddecode = fr_ragged.to_tensor(default_value=0, shape=[None, None, max_length])\n",
        "\n",
        "en_sequences = [en_tokenizer.tokenize(english_sentence.numpy().decode('utf-8')).merge_dims(-2,-1)\n",
        "                for _, english_sentence in dataset.take(127085)]\n",
        "en_ragged = tf.ragged.stack(en_sequences)\n",
        "en_padded = en_ragged.to_tensor(default_value=0, shape=[None, None, max_length])\n",
        "\n",
        "x_train = fr_padded\n",
        "x2_train = fr_paddeddecode\n",
        "y_train = en_padded\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "inputs = tf.keras.Input(shape=(1,200))\n",
        "embedding_dim = 256\n",
        "lstm_units = 512\n",
        "vocab_size_en = len(en_vocab) + len(reserved_tokens)\n",
        "vocab_size_fr = len(fr_vocab) + len(reserved_tokens)\n",
        "\n",
        "encoder_inputs = tf.keras.layers.Input(shape=(200,))\n",
        "encoder_embedding = tf.keras.layers.Embedding(input_dim=vocab_size_en, output_dim=embedding_dim, mask_zero=True)(encoder_inputs)\n",
        "encoder_outputs, state_h, state_c = tf.keras.layers.LSTM(lstm_units, return_state=True)(encoder_embedding)\n",
        "encoder_states = [state_h, state_c]\n",
        "\n",
        "decoder_inputs = tf.keras.layers.Input(shape=(200,))\n",
        "decoder_embedding = tf.keras.layers.Embedding(input_dim=vocab_size_fr, output_dim=embedding_dim, mask_zero=True)(decoder_inputs)\n",
        "decoder_lstm = tf.keras.layers.LSTM(lstm_units, return_sequences=True, return_state=True)\n",
        "decoder_outputs, _, _ = decoder_lstm(decoder_embedding, initial_state=encoder_states)\n",
        "decoder_dense = tf.keras.layers.Dense(vocab_size_fr, activation='softmax')\n",
        "decoder_outputs = decoder_dense(decoder_outputs)\n",
        "model = tf.keras.models.Model(inputs=[encoder_inputs, decoder_inputs], outputs=decoder_outputs)\n",
        "model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n",
        "\n",
        "\n",
        "model.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 493
        },
        "id": "FIp6S-MR7Pls",
        "outputId": "0a2d9241-1984-4aa7-c1d1-c5a8d982e542"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1mModel: \"functional_9\"\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"functional_9\"</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
              "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)             \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape          \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m       Param #\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mConnected to          \u001b[0m\u001