add tokenizer

added_tokens.json

@@ -0,0 +1 @@
+{"<INSTR_START>": 50274, "<TITLE_END>": 50270, "<INSTR_END>": 50276, "<INPUT_END>": 50268, "<INPUT_START>": 50266, "[PAD]": 50279, "<TITLE_START>": 50269, "<|endoftext|>": 50265, "<INGR_END>": 50273, "<NEXT_INPUT>": 50267, "<RECIPE_END>": 50278, "<RECIPE_START>": 50277, "<NEXT_INSTR>": 50275, "<INGR_START>": 50271, "<NEXT_INGR>": 50272}

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>", "pad_token": "[PAD]", "additional_special_tokens": ["<INPUT_START>", "<NEXT_INPUT>", "<INPUT_END>", "<TITLE_START>", "<TITLE_END>", "<INGR_START>", "<NEXT_INGR>", "<INGR_END>", "<INSTR_START>", "<NEXT_INSTR>", "<INSTR_END>", "<RECIPE_START>", "<RECIPE_END>"]}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "special_tokens_map_file": null, "name_or_path": "flax-community/gpt-2-spanish", "tokenizer_class": "GPT2Tokenizer"}