{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "3UeycYCyxDfE"
},
"source": [
"# TRANSLATOR"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "8BfUjVxBcz5N"
},
"source": [
"## instalation"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "WXqM38xBRHu2"
},
"outputs": [],
"source": [
"%%time\n",
"!pip install -q -U tensorflow-text\n",
"!pip install datasets\n",
"!pip install -q tensorflow_datasets\n",
"!pip install pydot\n",
"!cd /content\n",
"!clear"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"id": "Ukvs1XfMG7aG"
},
"outputs": [],
"source": [
"import tensorflow as tf\n",
"import tensorflow_text as tf_text\n",
"import tensorflow_datasets as tfds\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"import requests\n",
"import functools\n",
"import collections\n",
"import os\n",
"import pathlib\n",
"import re\n",
"import string\n",
"import tempfile\n",
"import time\n",
"import matplotlib.pyplot as plt\n",
"import os\n",
"import re\n",
"import shutil\n",
"import string\n",
"import tensorflow as tf\n",
"\n",
"from tensorflow.keras import layers\n",
"from tensorflow.keras import losses\n",
"import pydot"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "7V7igFwpc6Hs"
},
"source": [
"## dataset"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"id": "ZaMtoUtAREzs",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "67fb1777-9e10-4197-88dc-1187b08082ac"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Nombre de phrases en français : 127085\n",
"Nombre de phrases en anglais : 127085\n"
]
}
],
"source": [
"from datasets import load_dataset\n",
"\n",
"dataset = load_dataset("Helsinki-NLP/opus_books", "en-fr")\n",
"data = dataset["train"]\n",
"\n",
"french_sentences = [example["fr"] for example in data["translation"][:127085]]\n",
"english_sentences = [example["en"] for example in data["translation"][:127085]]\n",
"dataset = tf.data.Dataset.from_tensor_slices((french_sentences, english_sentences))\n",
"\n",
"french_sentences_decoded = []\n",
"english_sentences_decoded = []\n",
"\n",
"for french_sentence, english_sentence in dataset.take(127085):\n",
" french_sentences_decoded.append("b '"+french_sentence.numpy().decode('utf-8'))\n",
" english_sentences_decoded.append("b '"+english_sentence.numpy().decode('utf-8'))\n",
"\n",
"print("Nombre de phrases en français :", len(french_sentences_decoded))\n",
"print("Nombre de phrases en anglais :", len(english_sentences_decoded))"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"id": "2tb7H5uFQoBA"
},
"outputs": [],
"source": [
"train_fr = french_sentences\n",
"train_en = english_sentences"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "NzS8h0budHWv"
},
"source": [
"## vocab"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"id": "72uMXsQhFIx8"
},
"outputs": [],
"source": [
"from tensorflow_text.tools.wordpiece_vocab import bert_vocab_from_dataset as bert_vocab\n",
"\n",
"bert_tokenizer_params = dict(lower_case=True)\n",
"reserved_tokens = ["[PAD]", "[UNK]", "[START]", "[END]"]\n",
"\n",
"bert_vocab_args = {\n",
" 'vocab_size': 8000,\n",
" 'reserved_tokens': reserved_tokens,\n",
" 'bert_tokenizer_params': bert_tokenizer_params,\n",
" 'learn_params': {},\n",
"}\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "q6f3mrA-DK0n"
},
"outputs": [],
"source": [
"%%time\n",
"en_vocab = bert_vocab.bert_vocab_from_dataset(\n",
" tf.data.Dataset.from_tensor_slices(english_sentences).batch(1000).prefetch(2),\n",
" **bert_vocab_args\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"id": "EJZzz1x5YY0x",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "1e627917-410a-479f-d27d-ca31480d12bb"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"CPU times: user 3min 38s, sys: 1.51 s, total: 3min 39s\n",
"Wall time: 3min 40s\n"
]
}
],
"source": [
"%%time\n",
"fr_vocab = bert_vocab.bert_vocab_from_dataset(\n",
" tf.data.Dataset.from_tensor_slices(french_sentences).batch(1000).prefetch(2),\n",
" **bert_vocab_args\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"id": "1hmXuHHNcBHg"
},
"outputs": [],
"source": [
"def write_vocab_file(filepath, vocab):\n",
" with open(filepath, 'w') as f:\n",
" for token in vocab:\n",
" print(token, file=f)\n",
"write_vocab_file('en_vocab.txt', en_vocab)\n",
"write_vocab_file('fr_vocab.txt', fr_vocab)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "kLTf_mEvfNR9"
},
"source": [
"#TOKENIZER\n",
"\n",
"\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"id": "jGshnQ2idy8I"
},
"outputs": [],
"source": [
"fr_tokenizer = tf_text.BertTokenizer('fr_vocab.txt', **bert_tokenizer_params)\n",
"en_tokenizer = tf_text.BertTokenizer('en_vocab.txt', **bert_tokenizer_params)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"id": "bbQyYKhHkkDe",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "ef58d907-9286-471c-e286-2fa1cda19af9"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"[3092, 501, 202, 151, 107, 2890, 177, 4626]\n"
]
}
],
"source": [
"# Tokenize the examples -> (batch, word, word-piece)\n",
"en_tokenizere = en_tokenizer.tokenize("hello how are you Vadim")\n",
"# Merge the word and word-piece axes -> (batch, tokens)\n",
"en_tokenizere= en_tokenizere.merge_dims(-2,-1)\n",
"\n",
"for ex in en_tokenizere.to_list():\n",
" print(ex)\n"
]
},
{
"cell_type": "code",
"source": [
"words = en_tokenizer.detokenize(token_batch)\n",
"tf.strings.reduce_join(words, separator=' ', axis=-1)"
],
"metadata": {
"id": "k0m1461Gwy3e"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"## model"
],
"metadata": {
"id": "BjoPdwoxBWw2"
}
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"id": "7OqoDOybbwi6"
},
"outputs": [],
"source": [
"max_length = 200\n",
"\n",
"fr_sequences = [fr_tokenizer.tokenize(french_sentence.numpy().decode('utf-8')).merge_dims(-2,-1)\n",
" for french_sentence, _ in dataset.take(1000)]\n",
"fr_ragged = tf.ragged.stack(fr_sequences)\n",
"fr_padded = fr_ragged.to_tensor(default_value=0, shape=[None, None, max_length])\n",
"\n",
"fr_sequencesdeocde = [fr_tokenizer.tokenize("[START]"+french_sentence.numpy().decode('utf-8')+"[END]").merge_dims(-2,-1)\n",
" for french_sentence, _ in dataset.take(1000)]\n",
"fr_raggeddecode = tf.ragged.stack(fr_sequences)\n",
"fr_paddeddecode = fr_ragged.to_tensor(default_value=0, shape=[None, None, max_length])\n",
"\n",
"en_sequences = [en_tokenizer.tokenize(english_sentence.numpy().decode('utf-8')).merge_dims(-2,-1)\n",
" for _, english_sentence in dataset.take(1000)]\n",
"en_ragged = tf.ragged.stack(en_sequences)\n",
"en_padded = en_ragged.to_tensor(default_value=0, shape=[None, None, max_length])\n",
"\n",
"x_train = fr_padded\n",
"x2_train = fr_paddeddecode\n",
"y_train = en_padded\n",
"\n"
]
},
{
"cell_type": "code",
"source": [
"inputs = tf.keras.Input(shape=(1,200))\n",
"embedding_dim = 200\n",
"lstm_units = 512\n",
"vocab_size_en = len(en_vocab) + len(reserved_tokens)\n",
"vocab_size_fr = len(fr_vocab) + len(reserved_tokens)\n",
"\n",
"x = layers.LSTM(200, return_sequences=True)(inputs)\n",
"x = layers.Embedding(input_dim=vocab_size_fr, output_dim=embedding_dim, mask_zero=True)(x)\n",
"x = layers.LSTM(200, return_sequences=True)(x)\n",
"x = layers.LSTM(1024, return_sequences=True)(x)\n",
"x = layers.LSTM(500, return_sequences=True)(x)\n",
"x = layers.LSTM(400, return_sequences=True)(x)\n",
"x = layers.LSTM(300, return_sequences=True)(x)\n",
"outputs = layers.Dense(200)(x)\n",
"\n",
"model = tf.keras.Model(inputs=inputs, outputs=outputs)\n",
"model.summary()"
],
"metadata": {
"id": "NFo9fU5lw2IR"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "X--5AeJ7GgPp"
},
"outputs": [],
"source": [
"model.compile(tf.keras.optimizers.Adam(beta_1=0.9, beta_2=0.98,\n",
" epsilon=1e-9),\n",
" loss='categorical_crossentropy',\n",
" metrics=['accuracy'])"
]
},
{
"cell_type": "code",
"source": [
"inputs = tf.keras.Input(shape=(1,200))\n",
"embedding_dim = 200\n",
"lstm_units = 512\n",
"vocab_size_en = len(en_vocab) + len(reserved_tokens)\n",
"vocab_size_fr = len(fr_vocab) + len(reserved_tokens)\n",
"\n",
"encoder_inputs = tf.keras.layers.Input(shape=(200,))\n",
"encoder_embedding = tf.keras.layers.Embedding(input_dim=vocab_size_en, output_dim=embedding_dim, mask_zero=True)(encoder_inputs)\n",
"encoder_outputs, state_h, state_c = tf.keras.layers.LSTM(lstm_units, return_state=True)(encoder_embedding)\n",
"encoder_states = [state_h, state_c]\n",
"\n",
"decoder_inputs = tf.keras.layers.Input(shape=(200,))\n",
"decoder_embedding = tf.keras.layers.Embedding(input_dim=vocab_size_fr, output_dim=embedding_dim, mask_zero=True)(decoder_inputs)\n",
"decoder_lstm = tf.keras.layers.LSTM(lstm_units, return_sequences=True, return_state=True)\n",
"decoder_outputs, _, _ = decoder_lstm(decoder_embedding, initial_state=encoder_states)\n",



It looks like you're building a sequence-to-sequence model for English to French translation using TensorFlow and Keras. Here are a few suggestions to complete the model:

Add a dense output layer to the decoder to generate the predicted French words:

decoder_dense = tf.keras.layers.Dense(vocab_size_fr, activation='softmax')(decoder_outputs)
Define the model inputs and outputs:

model = tf.keras.models.Model(inputs=[encoder_inputs, decoder_inputs], outputs=decoder_dense)
Compile the model with an appropriate loss function and optimizer:

model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
Prepare the training data:

Convert the English and French sentences to integer sequences using the respective tokenizers.
Pad the sequences to a fixed length.
Split the data into training and validation sets.
Train the model:


model.fit([encoder_input_data, decoder_input_data], decoder_target_data,
          batch_size=batch_size,
          epochs=epochs,
          validation_split=0.2)
Evaluate the model on a test set and calculate metrics like BLEU score to assess the translation quality.

Use the trained model for inference:

Tokenize and encode the input English sentence.
Use the encoder to generate the initial states for the decoder.
Iteratively predict the French words using the decoder until an end token is generated or a maximum length is reached.
Decode the predicted French sequence back to text.
Here's an example of how the inference step could look:


def translate_sentence(sentence):
    # Tokenize and encode the input sentence
    input_seq = en_tokenizer.tokenize(sentence).merge_dims(-2,-1)
    input_seq = input_seq.to_tensor(default_value=0, shape=[None, max_length])
    
    # Generate the initial states for the decoder
    states_value = encoder_model.predict(input_seq)
    
    # Initialize the decoder input with the start token
    target_seq = np.zeros((1, 1))
    target_seq[0, 0] = fr_tokenizer.tokenize('[START]')[0]
    
    stop_condition = False
    decoded_sentence = ''
    while not stop_condition:
        output_tokens, h, c = decoder_model.predict([target_seq] + states_value)
        sampled_token_index = np.argmax(output_tokens[0, -1, :])
        sampled_token = fr_tokenizer.detokenize([sampled_token_index])
        
        if sampled_token == '[END]' or len(decoded_sentence.split()) > max_length:
            stop_condition = True
        else:
            decoded_sentence += sampled_token.numpy().decode('utf-8') + ' '
        
        target_seq = np.zeros((1, 1))
        target_seq[0, 0] = sampled_token_index
        states_value = [h, c]
    
    return decoded_sentence
These are the main steps to complete your English to French translation model. Make sure to handle text preprocessing, tokenization, and padding appropriately. Experiment with different model architectures, hyperparameters, and training strategies to improve the translation quality.



what i should replace by this cells ? :
inputs = tf.keras.Input(shape=(1,200))
embedding_dim = 200
lstm_units = 512
vocab_size_en = len(en_vocab) + len(reserved_tokens)
vocab_size_fr = len(fr_vocab) + len(reserved_tokens)

encoder_inputs = tf.keras.layers.Input(shape=(200,))
encoder_embedding = tf.keras.layers.Embedding(input_dim=vocab_size_en, output_dim=embedding_dim, mask_zero=True)(encoder_inputs)
encoder_outputs, state_h, state_c = tf.keras.layers.LSTM(lstm_units, return_state=True)(encoder_embedding)
encoder_states = [state_h, state_c]

decoder_inputs = tf.keras.layers.Input(shape=(200,))
decoder_embedding = tf.keras.layers.Embedding(input_dim=vocab_size_fr, output_dim=embedding_dim, mask_zero=True)(decoder_inputs)
decoder_lstm = tf.keras.layers.LSTM(lstm_units, return_sequences=True, return_state=True)
decoder_outputs, _, _ = decoder_lstm(decoder_embedding, initial_state=encoder_states)
decoder_dense = tf.keras.layers.Dense(vocab_size_fr, activation='softmax')
decoder_outputs = decoder_dense(decoder_outputs)

model = tf.keras.Model([encoder_inputs, decoder_inputs], decoder_outputs)
model.compile(optimizer=tf.keras.optimizers.Adam(), loss='sparse_categorical_crossentropy', metrics=['accuracy'])

model.summary()



Based on the code you provided, it seems like you have already implemented the necessary changes I suggested. Let's go through the code and make sure everything is in place:

You have defined the encoder inputs, embedding layer, and LSTM layer:

encoder_inputs = tf.keras.layers.Input(shape=(200,))
encoder_embedding = tf.keras.layers.Embedding(input_dim=vocab_size_en, output_dim=embedding_dim, mask_zero=True)(encoder_inputs)
encoder_outputs, state_h, state_c = tf.keras.layers.LSTM(lstm_units, return_state=True)(encoder_embedding)
encoder_states = [state_h, state_c]
You have defined the decoder inputs, embedding layer, LSTM layer, and dense output layer:

decoder_inputs = tf.keras.layers.Input(shape=(200,))
decoder_embedding = tf.keras.layers.Embedding(input_dim=vocab_size_fr, output_dim=embedding_dim, mask_zero=True)(decoder_inputs)
decoder_lstm = tf.keras.layers.LSTM(lstm_units, return_sequences=True, return_state=True)
decoder_outputs, _, _ = decoder_lstm(decoder_embedding, initial_state=encoder_states)
decoder_dense = tf.keras.layers.Dense(vocab_size_fr, activation='softmax')
decoder_outputs = decoder_dense(decoder_outputs)
You have defined the model inputs and outputs:

model = tf.keras.Model([encoder_inputs, decoder_inputs], decoder_outputs)
You have compiled the model with the appropriate loss function and optimizer:

model.compile(optimizer=tf.keras.optimizers.Adam(), loss='sparse_categorical_crossentropy', metrics=['accuracy'])
You have printed the model summary:

model.summary()
The code you provided looks good and follows the necessary steps to build the sequence-to-sequence model for translation. You don't need to replace anything in these cells.

The next steps would be to prepare your training data, train the model, evaluate its performance, and use it for inference as described in my previous response.

Remember to preprocess and tokenize your training data, pad the sequences to a fixed length, and split the data into training and validation sets before training the model. You can use the fit() method to train the model and pass the appropriate input and target data.

After training, you can evaluate the model on a test set and calculate metrics like BLEU score to assess the translation quality. Finally, you can use the trained model for inference by following the steps outlined in the previous response to translate new English sentences to French.



history = model.fit([x_train,x2_train], y_train, epochs=10, batch_size=2)
Epoch 1/10
ValueError Traceback (most recent call last)
in <cell line: 1>()
----> 1 history = model.fit([x_train,x2_train], y_train, epochs=10, batch_size=2)

1 frames
/usr/local/lib/python3.10/dist-packages/keras/src/utils/traceback_utils.py in error_handler(*args, **kwargs)
120 # To get the full stack trace, call:
121 # keras.config.disable_traceback_filtering()
--> 122 raise e.with_traceback(filtered_tb) from None
123 finally:
124 del filtered_tb

/usr/local/lib/python3.10/dist-packages/keras/src/layers/input_spec.py in assert_input_compatibility(input_spec, inputs, layer_name)
243 if spec_dim is not None and dim is not None:
244 if spec_dim != dim:
--> 245 raise ValueError(
246 f'Input {input_index} of layer "{layer_name}" is '
247 "incompatible with the layer: "

ValueError: Input 0 of layer "functional_3" is incompatible with the layer: expected shape=(None, 200), found shape=(2, 1, 200)



The error message suggests that there is a mismatch between the expected input shape of the model and the actual shape of the input data you are providing.

The model expects the input shape to be (None, 200), but the shape of x_train is (2, 1, 200). It seems like x_train has an extra dimension of size 1.

To resolve this issue, you can try reshaping x_train and x2_train to remove the extra dimension. You can use the reshape() function from NumPy to do this:


x_train = x_train.reshape((x_train.shape[0], x_train.shape[-1]))
x2_train = x2_train.reshape((x2_train.shape[0], x2_train.shape[-1]))
This will reshape x_train and x2_train to have the shape (num_samples, 200), which should match the expected input shape of the model.

After reshaping the input data, you can try running the fit() method again:


history = model.fit([x_train, x2_train], y_train, epochs=10, batch_size=2)
Make sure that y_train also has the appropriate shape (num_samples, 200) and matches the output shape of the model.

If you still encounter any shape mismatch errors, double-check the shapes of your input and target data and ensure they align with the model's expected shapes.

Additionally, ensure that the number of samples in x_train, x2_train, and y_train are the same.

If the issue persists, please provide more information about the shapes of your input and target data, and I'll be happy to assist you further.



no can you change the model for accepted data like x1 and x2 now