max_length = 200
def process_sentence(sentence, tokenizer, add_start_end=False):
   sentence_text = sentence.numpy().decode('utf-8')
   if add_start_end:
       sentence_text = "[START]" + sentence_text + "[END]"
   tokenized_sentence = tokenizer.tokenize(sentence_text).merge_dims(-2, -1)
   return tokenized_sentence

def tokenize_map_fn(french_sentence, english_sentence):
   fr_sentence_proc = tf.py_function(func=process_sentence, inp=[french_sentence, fr_tokenizer, False], Tout=tf.int64)
   fr_sentence_proc_deocde = tf.py_function(func=process_sentence, inp=[french_sentence, fr_tokenizer, True], Tout=tf.int64)
   en_sentence_proc = tf.py_function(func=process_sentence, inp=[english_sentence, en_tokenizer, False], Tout=tf.int64)
   return fr_sentence_proc, fr_sentence_proc_deocde, en_sentence_proc

# Assuming dataset already defined and is a tf.data.Dataset object
processed_dataset = dataset.take(127085).map(tokenize_map_fn)

# To efficiently create padded and ragged tensors, first stack all elements from processed_dataset
fr_sequences, fr_sequences_decode, en_sequences = [], [], []
for fr_seq, fr_seq_decode, en_seq in processed_dataset:
   fr_sequences.append(fr_seq)
   fr_sequences_decode.append(fr_seq_decode)
   en_sequences.append(en_seq)

# Convert lists of sequences into ragged tensors and then pad them
fr_ragged = tf.ragged.stack(fr_sequences)
fr_padded = fr_ragged.to_tensor(default_value=0, shape=[None, None, max_length])
print("fr sequences YES")

fr_ragged_decode = tf.ragged.stack(fr_sequences_decode)
fr_padded_decode = fr_ragged_decode.to_tensor(default_value=0, shape=[None, None, max_length])
print("sequencesdeocde Yes")

en_ragged = tf.ragged.stack(en_sequences)
en_padded = en_ragged.to_tensor(default_value=0, shape=[None, None, max_length])
print("en_sequences yes")

x_train = fr_padded
x2_train = fr_padded_decode
y_train = en_padded

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[29], line 16
     13    return fr_sentence_proc, fr_sentence_proc_deocde, en_sentence_proc
     15 # Assuming dataset already defined and is a tf.data.Dataset object
---> 16 processed_dataset = dataset.take(127085).map(tokenize_map_fn)
     18 # To efficiently create padded and ragged tensors, first stack all elements from processed_dataset
     19 fr_sequences, fr_sequences_decode, en_sequences = [], [], []

File ~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\tensorflow\python\data\ops\dataset_ops.py:2202, in DatasetV2.map(self, map_func, num_parallel_calls, deterministic, name)
   2199   if deterministic is not None and not DEBUG_MODE:
   2200     warnings.warn("The `deterministic` argument has no effect unless the "
   2201                   "`num_parallel_calls` argument is specified.")
-> 2202   return MapDataset(self, map_func, preserve_cardinality=True, name=name)
   2203 else:
   2204   return ParallelMapDataset(
   2205       self,
   2206       map_func,
   (...)
   2209       preserve_cardinality=True,
   2210       name=name)

File ~\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\tensorflow\python\data\ops\dataset_ops.py:5400, in MapDataset.__init__(self, input_dataset, map_func, use_inter_op_parallelism, preserve_cardinality, use_legacy_function, name)
   5398 self._use_inter_op_parallelism = use_inter_op_parallelism
   5399 self._preserve_cardinality = preserve_cardinality
...
    File "C:\Users\ixelo\AppData\Local\Temp\ipykernel_14408\98278579.py", line 10, in tokenize_map_fn  *
        fr_sentence_proc = tf.py_function(func=process_sentence, inp=[french_sentence, fr_tokenizer, False], Tout=tf.int64)

    TypeError: Tensors in list passed to 'input' of 'EagerPyFunc' Op have types [string, <NOT CONVERTIBLE TO TENSOR>, bool] that are invalid. Tensors: [<tf.Tensor 'args_0:0' shape=() dtype=string>, <tensorflow_text.python.ops.bert_tokenizer.BertTokenizer object at 0x000001CAAA3B6FE0>, False]
Output is truncated. View as a scrollable element or open in a text editor. Adjust cell output settings...