Spaces:
Running
Running
| import streamlit as st | |
| from transformers import ( | |
| AutoTokenizer, | |
| XLNetTokenizer | |
| ) | |
| import pathlib | |
| import json | |
| st.set_page_config(layout='wide') | |
| st.title("Transformers library For NLP Tasks : Structured by Topics") | |
| st.write("lets start with the architectures of models") | |
| neural_net_models = dict({ | |
| 'encoder': "responsible for understanding the input text.", | |
| 'decoder': "designed to generate new texts answering queries.", | |
| 'encoder-decoder': "understand and generate text & have emergent behaviour", | |
| 'convolution': "used for image recognition and processing.", | |
| }) | |
| model_types = list(neural_net_models.keys()) | |
| archs = st.radio("model architectures".capitalize(), model_types) | |
| st.write(f"{archs.capitalize()} are {neural_net_models[archs]}") | |
| domains = dict({ | |
| "computer_vision": { | |
| "encoder": ['vit', 'swin', 'segformer', 'beit'], | |
| "decoder": ['imagegpt'], | |
| "encoder-decoder": ['detr'], | |
| "convolution": ['convnext'] | |
| }, | |
| "nlp": { | |
| "encoder": ["bert", "roberta", "albert", "distillbert", | |
| "deberta", "longformer",], | |
| "decoder": ["gpt-2", "xlnet", "gpt-j", "opt", "bloom"], | |
| "encoder-decoder": ["bart", "pegasus", "t5", ], | |
| }, | |
| "audio": { | |
| "encoder": ["wav2vec2", "hubert"], | |
| "encoder-decoder": ["speech2text", "whisper"] | |
| }, | |
| "multimodal": { | |
| "encoder": ["visualbert", "vilt", "clip", "owl-vit"], | |
| "encoder-decoder": ["trocr", "donut"] | |
| }, | |
| "reinforcement": { | |
| "decoder": ["trajectory transformer", "decision transformer"] | |
| } | |
| }) | |
| st.write("Lets look at the Individual domains") | |
| domain_list = list(domains.keys()) | |
| doms = st.radio("domains of ai".capitalize(), domain_list) | |
| st.write(domains[doms]) | |
| st.write("Now comes the Tokenizers, the Entry Points") | |
| tokenizer_algos = { | |
| "byte_pair": { | |
| "base": ['gpt', 'gpt-2(byte_level)'], | |
| "intro": "https://arxiv.org/abs/1508.07909" | |
| }, | |
| "wordpiece":{ | |
| "base": ['bert', 'distilbert', 'electra'], | |
| "intro": "https://static.googleusercontent.com/media/research.google.com/ja//pubs/archive/37842.pdf" | |
| }, | |
| "unigram": { | |
| "base": ['not_used'], | |
| "intro": "https://arxiv.org/pdf/1804.10959.pdf" | |
| }, | |
| "sentencepiece":{ | |
| "base": ["xlm", "albert", "xlnet", "marian", "t5"], | |
| "intro": "https://arxiv.org/pdf/1808.06226.pdf" | |
| } | |
| } | |
| tokenizer_items = list(tokenizer_algos.keys()) | |
| algos = st.radio("tokenizer algos".capitalize(), tokenizer_items) | |
| st.write(tokenizer_algos[algos]) | |
| st.write("""We will work on 3 types of tokenizers on a single sentence | |
| to see how their output differs, by first encoding and decoding them too.""") | |
| st.markdown("""### Models in Review: | |
| - gpt2 | |
| - bert-base-uncased | |
| - xlm""") | |
| input_sentence = "This is a sample sentence for testing tokenizers" | |
| gpt2_model = "gpt2" | |
| bert_model = "bert-base-uncased" | |
| xlm_model = "xlnet-base-cased" | |
| gpt2_tokenizer = AutoTokenizer.from_pretrained(gpt2_model) | |
| bert_tokenizer = AutoTokenizer.from_pretrained(bert_model) | |
| xlm_tokenizer = XLNetTokenizer.from_pretrained(xlm_model) | |
| st.markdown("#### The input sentence is") | |
| st.write("The Sample Sentence: ", input_sentence) | |
| gpt2_tokenize = gpt2_tokenizer.tokenize(input_sentence) | |
| bert_tokenize = bert_tokenizer.tokenize(input_sentence) | |
| xlm_tokenize = xlm_tokenizer.tokenize(input_sentence) | |
| with st.expander(label="Byte Pair Tokenizer", expanded=False): | |
| st.write("gpt2_tokenize = gpt2_tokenizer.tokenize(input_sentence)") | |
| st.write(gpt2_tokenize) | |
| with st.expander(label="Word Piece Tokenizer", expanded=False): | |
| st.write("bert_tokenize = bert_tokenizer.tokenize(input_sentence)") | |
| st.write(bert_tokenize) | |
| with st.expander(label="SentencePiece Tokenizer", expanded=False): | |
| st.write("xlm_tokenize = xlm_tokenizer.tokenize(input_sentence)") | |
| st.write(xlm_tokenize) | |
| st.markdown("""#### Tokenizer Options: | |
| There are following parameters in Tokenizer object are most used | |
| - padding = 'longest'(True), 'max_length', 'do_not_pad'(False) | |
| - truncation = 'longest_first'(True), 'only_second', 'only_first', | |
| 'do_not_truncate'(False) | |
| - max_length = <= model_max_length """) | |
| ## Refer to https://huggingface.co/docs/transformers/pad_truncation | |
| gpt2_max_length = gpt2_tokenizer.model_max_length | |
| bert_max_length = bert_tokenizer.model_max_length | |
| xlm_max_length = "Not Speced" | |
| st.markdown("""We also need the model max length, which is the | |
| what the model is configured with.""") | |
| st.write("GPT: ", gpt2_max_length) | |
| st.write("Bert: ", bert_max_length) | |
| st.write("XLM: ", xlm_max_length) | |
| sent1 = "This app is talking about the variety of Tokenizers and their outputs" | |
| sent2 = """Tokenizers do one thing, bring out numbers from text. The better numbers far better | |
| the results""" | |
| st.write("We will be working with the following sentences.") | |
| st.write("Sentence1: ", sent1) | |
| st.write("Sentence2: ", sent2) | |
| st.markdown("#### Tokenization in Action. Using GPT Tokenizer") | |
| st.markdown("""##### Trial-1: | |
| > No parameter provided | |
| > Sentences are given with comma seperation""") | |
| gpt2_encode = gpt2_tokenizer(sent1, sent2) | |
| st.write(gpt2_encode) | |
| st.markdown("""##### Trial-2: | |
| > No parameter provided | |
| > Sentences are made into a List""") | |
| gpt2_encode = gpt2_tokenizer([sent1, sent2]) | |
| st.write("gpt2_encode = gpt2_tokenizer([sent1, sent2])") | |
| st.write(gpt2_encode) | |
| # gpt2_tokenizer.add_special_tokens({'pad_token': '[PAD]'}) | |
| gpt2_tokenizer.pad_token_id = gpt2_tokenizer.eos_token_id | |
| st.markdown("""##### Trial-3: | |
| > Need to add pad token to tokenizer, if the model doesn't have. | |
| > padding = True | |
| > Sentences are made into a List""") | |
| gpt2_encode = gpt2_tokenizer([sent1, sent2], padding=True) | |
| st.write("gpt2_encode = gpt2_tokenizer([sent1, sent2], padding=True)") | |
| st.write(gpt2_encode) | |
| st.markdown("""##### Trial-4: | |
| > Need to add pad token to tokenizer, if the model doesn't have. | |
| > padding = max_length (requires max_length = int) | |
| > Sentences are made into a List""") | |
| gpt2_encode = gpt2_tokenizer([sent1, sent2], | |
| padding=True, | |
| max_length=15) | |
| st.write("""gpt2_encode = gpt2_tokenizer([sent1, sent2], | |
| padding=True, | |
| max_length=15""") | |
| st.write(gpt2_encode) | |
| st.markdown("""##### Trial-5: | |
| > truncate = True (requires max_length = int) | |
| > Sentences are seperated by a comma | |
| Will see total output of 12 token, 6 per sentence""") | |
| gpt2_encode = gpt2_tokenizer(sent1, sent2, | |
| truncation=True, | |
| max_length=12) | |
| st.write("""gpt2_encode = gpt2_tokenizer(sent1, sent2, | |
| truncation=True, | |
| max_length=12)""") | |
| st.write(gpt2_encode) | |
| st.markdown("""##### Trial-6: | |
| > truncate = True (requires max_length = int) | |
| > Sentences are made into a list | |
| Will have longest first""") | |
| gpt2_encode = gpt2_tokenizer([sent1, sent2], | |
| truncation=True, | |
| max_length=12) | |
| st.write("""gpt2_encode = gpt2_tokenizer([sent1, sent2], | |
| truncation=True, | |
| max_length=12)""") | |
| st.write(gpt2_encode) | |
| st.markdown("""##### Trial-7: | |
| > truncate = only_first | |
| > Sentences are made into a list | |
| Will have only 8 tokens """) | |
| gpt2_encode = gpt2_tokenizer([sent1, sent2], | |
| truncation='only_first', | |
| max_length=8) | |
| st.write("""gpt2_encode = gpt2_tokenizer([sent1, sent2], | |
| truncation='only_first', | |
| max_length=8)""") | |
| st.write(gpt2_encode) | |
| st.markdown("""##### Trial-8: | |
| > truncate = False (only_second, is erroring out) | |
| > Sentences are made into a list | |
| No Truncation, 2 ids list""") | |
| gpt2_encode = gpt2_tokenizer([sent1, sent2], | |
| truncation=False, | |
| max_length=7) | |
| st.write("""gpt2_encode = gpt2_tokenizer([sent1, sent2], | |
| truncation=False, | |
| max_length=7)""") | |
| st.write(gpt2_encode) | |
| curr_dir = pathlib.Path(__file__).parent.resolve() | |
| file_loc = curr_dir / "task_arch.json" | |
| file_loc = file_loc.resolve() | |
| with open(file_loc, 'r') as arch: | |
| data = json.load(arch) | |
| tasks = list(data.keys()) | |
| st.markdown("#### Lets dive into the model architectures...") | |
| task = st.radio("The NLP tasks", tasks) | |
| task_data = data[task] | |
| num_models = len(task_data['architectures']) | |
| show_archs = st.slider("How many archs to Show", | |
| min_value=4, max_value=num_models) | |
| pruned_data = { | |
| "architectures": task_data['architectures'][:show_archs], | |
| "AutoModelClass": task_data["AutoModelClass"], | |
| "dataset": task_data["dataset"], | |
| "model_used": task_data["model_used"] | |
| } | |
| st.write(pruned_data) |