add FIX TOKENIZER! instructions

README.md

@@ -107,8 +107,6 @@ language:
 
 # `flores101_mm100_175M`
 
-https://www.statmt.org/wmt21/large-scale-multilingual-translation-task.html
-
 `flores101_mm100_175M` is a multilingual encoder-decoder (seq-to-seq) model trained for Many-to-Many multilingual translation. It was first released in [this](https://github.com/facebookresearch/fairseq/tree/main/examples/flores101) repository.
 
 ```python
@@ -117,8 +115,14 @@ from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer
 hi_text = "जीवन एक चॉकलेट बॉक्स की तरह है।"
 chinese_text = "生活就像一盒巧克力。"
 
-model = M2M100ForConditionalGeneration.from_pretrained("facebook/m2m100_418M")
-tokenizer = M2M100Tokenizer.from_pretrained("facebook/m2m100_418M")
+model = M2M100ForConditionalGeneration.from_pretrained("seyoungsong/flores101_mm100_175M")
+tokenizer: M2M100Tokenizer = M2M100Tokenizer.from_pretrained("seyoungsong/flores101_mm100_175M")
+
+# FIX TOKENIZER!
+tokenizer.lang_token_to_id = {t: i for t, i in zip(tokenizer.all_special_tokens, tokenizer.all_special_ids) if i > 5}
+tokenizer.lang_code_to_token = {s.strip("_"): s for s in tokenizer.lang_token_to_id}
+tokenizer.lang_code_to_id = {s.strip("_"): i for s, i in tokenizer.lang_token_to_id.items()}
+tokenizer.id_to_lang_token = {i: s for s, i in tokenizer.lang_token_to_id.items()}
 
 # translate Hindi to French
 tokenizer.src_lang = "hi"
@@ -132,7 +136,7 @@ tokenizer.src_lang = "zh"
 encoded_zh = tokenizer(chinese_text, return_tensors="pt")
 generated_tokens = model.generate(**encoded_zh, forced_bos_token_id=tokenizer.get_lang_id("en"))
 tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)
-# => "Life is like a box of chocolate."
+# => "Life is like a chocolate box."
 ```
 
 ## Languages covered