Update README.md

README.md

@@ -39,26 +39,6 @@ model = AutoModelForSeq2SeqLM.from_pretrained('cifope/nllb-200-wo-fr-distilled-6
 tokenizer = NllbTokenizer.from_pretrained('facebook/nllb-200-distilled-600M')
 ```
 
-## Tokenizer Customization
-
-To integrate specific features like new language codes into the tokenizer, you can use the `fix_tokenizer` function:
-
-```python
-def fix_tokenizer(tokenizer, new_lang='wol_Wol'):
-    old_len = len(tokenizer) - int(new_lang in tokenizer.added_tokens_encoder)
-    tokenizer.lang_code_to_id[new_lang] = old_len-1
-    tokenizer.id_to_lang_code[old_len-1] = new_lang
-    tokenizer.fairseq_tokens_to_ids["<mask>"] = len(tokenizer.sp_model) + len(tokenizer.lang_code_to_id) + tokenizer.fairseq_offset
-    tokenizer.fairseq_tokens_to_ids.update(tokenizer.lang_code_to_id)
-    tokenizer.fairseq_ids_to_tokens = {v: k for k, v in tokenizer.fairseq_tokens_to_ids.items()}
-    if new_lang not in tokenizer._additional_special_tokens:
-        tokenizer._additional_special_tokens.append(new_lang)
-    tokenizer.added_tokens_encoder = {}
-    tokenizer.added_tokens_decoder = {}
-
-fix_tokenizer(tokenizer)
-```
-
 ## Translation Functions
 
 ### Translate from French to Wolof
@@ -66,7 +46,7 @@ fix_tokenizer(tokenizer)
 The `translate` function translates text from French to Wolof:
 
 ```python
-def translate(text, src_lang='fra_Latn', tgt_lang='wol_Wol', a=16, b=1.5, max_input_length=1024, **kwargs):
+def translate(text, src_lang='fra_Latn', tgt_lang='wol_Latn', a=16, b=1.5, max_input_length=1024, **kwargs):
     tokenizer.src_lang = src_lang
     tokenizer.tgt_lang = tgt_lang
     inputs = tokenizer(text, return_tensors='pt', padding=True, truncation=True, max_length=max_input_length)
@@ -84,7 +64,7 @@ def translate(text, src_lang='fra_Latn', tgt_lang='wol_Wol', a=16, b=1.5, max_in
 The `reversed_translate` function translates text from Wolof to French:
 
 ```python
-def reversed_translate(text, src_lang='wol_Wol', tgt_lang='fra_Latn', a=16, b=1.5, max_input_length=1024, **kwargs):
+def reversed_translate(text, src_lang='wol_Latn', tgt_lang='fra_Latn', a=16, b=1.5, max_input_length=1024, **kwargs):
     tokenizer.src_lang = src_lang
     tokenizer.tgt_lang = tgt_lang
     inputs = tokenizer(text, return_tensors='pt', padding=True, truncation=True, max_length=max_input_length)
@@ -109,5 +89,8 @@ print(wolof_translation)
 wolof_text = "alkaati yi tàmbali nañu xàll léegi kilifa gi ñów"
 french_translation = reversed_translate(wolof_text)
 print(french_translation)
-```
 
+wolof_text = "alkaati yi tàmbali nañu xàll léegi kilifa gi ñów"
+english_translation = reversed_translate(wolof_text,tgt_lang="eng_Latn")
+print(english_translation)
+```