| from idiomify.fetchers import fetch_tokenizer | |
| def main(): | |
| tokenizer = fetch_tokenizer("t-1-1") | |
| sent = "There will always be a <idiom> silver lining </idiom> even when things look pitch black" | |
| ids = tokenizer(sent)['input_ids'] | |
| print(ids) | |
| decoded = tokenizer.decode(ids) | |
| print(decoded) | |
| if __name__ == '__main__': | |
| main() | |