This time for sure x4

app_context.py

@@ -1,258 +1,254 @@
-import gradio as gr
-import math
-import spacy
-from datasets import load_dataset
-from sentence_transformers import SentenceTransformer
-from sentence_transformers import InputExample
-from sentence_transformers import losses
-from sentence_transformers import util
-from transformers import pipeline, T5Tokenizer
-from transformers import AutoTokenizer, AutoModel, AutoModelForSequenceClassification
-from transformers import TrainingArguments, Trainer, T5ForConditionalGeneration
-import torch
-import torch.nn.functional as F
-from torch.utils.data import DataLoader
-import numpy as np
-import evaluate
-import nltk
-from nltk.corpus import stopwords
-import subprocess
-import sys
-import random
-from textwrap import fill
-
-# !pip install https://huggingface.co/spacy/en_core_web_sm/resolve/main/en_core_web_sm-any-py3-none-any.whl
-subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'https://huggingface.co/spacy/en_core_web_sm/resolve/main/en_core_web_sm-any-py3-none-any.whl'])
-# tokenizer = AutoTokenizer.from_pretrained('sentence-transformers/all-MiniLM-L6-v2')
-model_base = "results/checkpoint-17000"
-nltk.download('stopwords')
-nlp = spacy.load("en_core_web_sm")
-stops = stopwords.words("english")
-ROMAN_CONSTANTS = (
-            ( "", "I", "II", "III", "IV", "V", "VI", "VII", "VIII", "IX" ),
-            ( "", "X", "XX", "XXX", "XL", "L", "LX", "LXX", "LXXX", "XC" ),
-            ( "", "C", "CC", "CCC", "CD", "D", "DC", "DCC", "DCCC", "CM" ),
-            ( "", "M", "MM", "MMM", "",   "",  "-",  "",    "",     ""   ),
-            ( "", "i", "ii", "iii", "iv", "v", "vi", "vii", "viii", "ix" ),
-            ( "", "x", "xx", "xxx", "xl", "l", "lx", "lxx", "lxxx", "xc" ),
-            ( "", "c", "cc", "ccc", "cd", "d", "dc", "dcc", "dccc", "cm" ),
-            ( "", "m", "mm", "mmm", "",   "",  "-",  "",    "",     ""   ),
-        )
-
-# answer = "Pizza"
-guesses = []
-return_guesses = []
-answer = "Moon"
-word1 = "Black"
-word2 = "White"
-word3 = "Sun"
-base_prompts = ["Sun is to Moon as ", "Black is to White as ", "Atom is to Element as",
-                "Athens is to Greece as ", "Cat is to Dog as ", "Robin is to Bird as",
-                "Hunger is to Ambition as "]
-
-
-#Mean Pooling - Take attention mask into account for correct averaging
-def mean_pooling(model_output, attention_mask):
-    token_embeddings = model_output['token_embeddings'] #First element of model_output contains all token embeddings
-    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
-    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
-
-
-def normalize(comment, lowercase, remove_stopwords):
-    if lowercase:
-        comment = comment.lower()
-    comment = nlp(comment)
-    lemmatized = list()
-    for word in comment:
-        lemma = word.lemma_.strip()
-        if lemma:
-            if not remove_stopwords or (remove_stopwords and lemma not in stops):
-                lemmatized.append(lemma)
-    return " ".join(lemmatized)
-
-
-# def tokenize_function(examples):
-#     return tokenizer(examples["text"])
-
-
-def compute_metrics(eval_pred):
-    logits, labels = eval_pred
-    predictions = np.argmax(logits, axis=-1)
-    metric = evaluate.load("accuracy")
-    return metric.compute(predictions=predictions, references=labels)
-    
-    
-def get_model():
-    global model_base
-    # last_checkpoint = "./results/checkpoint-22500"
-
-    finetuned_model = T5ForConditionalGeneration.from_pretrained(model_base)
-    tokenizer = T5Tokenizer.from_pretrained(model_base)
-    # model = SentenceTransformer(model_base)
-    gpu_available = torch.cuda.is_available()
-    device = torch.device("cuda" if gpu_available else "cpu")
-    finetuned_model = finetuned_model.to(device)
-    return finetuned_model, tokenizer
-
-
-def cosine_scores(model, sentence):
-    global word1
-    global word2
-    global word3
-    # sentence1 = f"{word1} is to {word2} as"
-    embeddings1 = model.encode(sentence, convert_to_tensor=True)
-
-def embeddings(model, sentences, tokenizer):
-    global word1
-    global word2
-    global word3
-    global model_base
-    gpu_available = torch.cuda.is_available()
-    device = torch.device("cuda" if gpu_available else "cpu")
-    # device = torch.device('cuda:0')
-    # embeddings = model.encode(sentences)
-    question = "Please answer to this question: " + sentences
-    
-    inputs = tokenizer(question, return_tensors="pt")
-    
-    print(inputs)
-    # print(inputs.device)
-    print(model.device)
-    print(inputs['input_ids'].device)
-    print(inputs['attention_mask'].device)
-    
-    inputs['attention_mask'] = inputs['attention_mask'].to(device)
-    inputs['input_ids'] = inputs['input_ids'].to(device)
-    
-    outputs = model.generate(**inputs)
-    answer = tokenizer.decode(outputs[0])
-    answer = answer[6:-4]
-    # print(fill(answer, width=80))
-    
-    print("ANSWER IS", answer)
-
-    return answer
-
-
-def random_word(model, tokenizer):
-    global model_base
-    vocab = tokenizer.get_vocab()
-    # with open(model_base + '/vocab.txt', 'r') as file:
-    line = ""
-    # content = file.readlines()
-    length = tokenizer.vocab_size
-    # print(vocab)
-    while line == "":
-        rand_line = random.randrange(0, length)
-        # print("TRYING TO FIND", rand_line, "OUT OF", length, "WITH VOCAB OF TYPE", type(vocab))
-        for word, id in vocab.items():
-            if id == rand_line and word[0].isalpha() and word not in stops and word not in ROMAN_CONSTANTS:
-        # if vocab[rand_line][0].isalpha() and vocab[rand_line][:-1] not in stops and vocab[rand_line][:-1] not in ROMAN_CONSTANTS:
-                line = word
-            elif id == rand_line:
-                print(f"{word} is not alpha or is a stop word")
-    # for num, aline in enumerate(file, 1997):
-    #     if random.randrange(num) and aline.isalpha():
-    #         continue
-    #     # elif not aline.isalpha():
-            
-    #     line = aline
-    print(line)
-    return line
-
-
-def generate_prompt(model, tokenizer):
-    global word1
-    global word2
-    global word3
-    global answer
-    global base_prompts
-    word1 = random_word(model, tokenizer)
-    # word2 = random_word()
-    
-    word2 = embeddings(model, f"{base_prompts[random.randint(0, len(base_prompts) - 1)]}{word1} is to ___.", tokenizer)
-    word3 = random_word(model, tokenizer)
-    sentence = f"{word1} is to {word2} as {word3} is to ___."
-    print(sentence)
-    answer = embeddings(model, sentence, tokenizer)
-    print("ANSWER IS", answer)
-    return f"# {word1} is to {word2} as {word3} is to ___."
-    # cosine_scores(model, sentence)
-
-
-def greet(name):
-    return "Hello " + name + "!!"
-
-def check_answer(guess:str):
-    global guesses
-    global answer
-    global return_guesses
-    global word1
-    global word2
-    global word3
-    
-    model, tokenizer = get_model()
-    output = ""
-    protected_guess = guess
-    sentence = f"{word1} is to {word2} as [MASK] is to {guess}."
-   
-    other_word = embeddings(model, sentence, tokenizer)
-    guesses.append(guess)
-    
-    
-    
-    for guess in return_guesses:
-        output += ("- " + guess + "<br>")
-    
-    # output = output[:-1]
-    prompt = f"{word1} is to {word2} as {word3} is to ___."
-    # print("IS", protected_guess, "EQUAL TO", answer, ":", protected_guess.lower() == answer.lower())
-    
-    if protected_guess.lower() == answer.lower():
-        return_guesses.append(f"{protected_guess}: {word1} is to {word2} as {word3} is to {protected_guess}.")
-        output += f"<span style='color:green'>- {return_guesses[-1]}</span><br>"
-        new_prompt = generate_prompt(model, tokenizer)
-        return new_prompt, "Correct!", output
-    else:
-        return_guess = f"{protected_guess}: {word1} is to {word2} as {other_word} is to {protected_guess}."
-        return_guesses.append(return_guess)
-        output += ("- " + return_guess + " <br>")
-        return prompt, "Try again!", output
-
-def main():
-    global word1
-    global word2
-    global word3
-    global answer
-    # answer = "Moon"
-    global guesses
-    
-    
-    # num_rows, data_type, value, example, embeddings = training()
-    # sent_embeddings = embeddings()
-    model, tokenizer = get_model() 
-    generate_prompt(model, tokenizer)
-    
-    prompt = f"{word1} is to {word2} as {word3} is to ____"
-    print(prompt)
-    print("TESTING EMBEDDINGS")
-    with gr.Blocks() as iface:
-        mark_question = gr.Markdown(prompt)
-        with gr.Tab("Guess"):
-            text_input = gr.Textbox()
-            text_output = gr.Textbox()
-            text_button = gr.Button("Submit")
-        with gr.Accordion("Open for previous guesses"):
-            text_guesses = gr.Markdown()
-        # with gr.Tab("Testing"):
-        #     gr.Markdown(f"""The Embeddings are {sent_embeddings}.""")
-        text_button.click(check_answer, inputs=[text_input], outputs=[mark_question, text_output, text_guesses])
-    # iface = gr.Interface(fn=greet, inputs="text", outputs="text")
-    iface.launch()
-    
-    
-
-
-    
-if __name__ == "__main__":
+import gradio as gr
+import math
+import spacy
+from datasets import load_dataset
+from transformers import pipeline, T5Tokenizer
+from transformers import AutoTokenizer, AutoModel, AutoModelForSequenceClassification
+from transformers import TrainingArguments, Trainer, T5ForConditionalGeneration
+import torch
+import torch.nn.functional as F
+from torch.utils.data import DataLoader
+import numpy as np
+import evaluate
+import nltk
+from nltk.corpus import stopwords
+import subprocess
+import sys
+import random
+from textwrap import fill
+
+# !pip install https://huggingface.co/spacy/en_core_web_sm/resolve/main/en_core_web_sm-any-py3-none-any.whl
+subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'https://huggingface.co/spacy/en_core_web_sm/resolve/main/en_core_web_sm-any-py3-none-any.whl'])
+# tokenizer = AutoTokenizer.from_pretrained('sentence-transformers/all-MiniLM-L6-v2')
+model_base = "results/checkpoint-17000"
+nltk.download('stopwords')
+nlp = spacy.load("en_core_web_sm")
+stops = stopwords.words("english")
+ROMAN_CONSTANTS = (
+            ( "", "I", "II", "III", "IV", "V", "VI", "VII", "VIII", "IX" ),
+            ( "", "X", "XX", "XXX", "XL", "L", "LX", "LXX", "LXXX", "XC" ),
+            ( "", "C", "CC", "CCC", "CD", "D", "DC", "DCC", "DCCC", "CM" ),
+            ( "", "M", "MM", "MMM", "",   "",  "-",  "",    "",     ""   ),
+            ( "", "i", "ii", "iii", "iv", "v", "vi", "vii", "viii", "ix" ),
+            ( "", "x", "xx", "xxx", "xl", "l", "lx", "lxx", "lxxx", "xc" ),
+            ( "", "c", "cc", "ccc", "cd", "d", "dc", "dcc", "dccc", "cm" ),
+            ( "", "m", "mm", "mmm", "",   "",  "-",  "",    "",     ""   ),
+        )
+
+# answer = "Pizza"
+guesses = []
+return_guesses = []
+answer = "Moon"
+word1 = "Black"
+word2 = "White"
+word3 = "Sun"
+base_prompts = ["Sun is to Moon as ", "Black is to White as ", "Atom is to Element as",
+                "Athens is to Greece as ", "Cat is to Dog as ", "Robin is to Bird as",
+                "Hunger is to Ambition as "]
+
+
+#Mean Pooling - Take attention mask into account for correct averaging
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output['token_embeddings'] #First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+
+def normalize(comment, lowercase, remove_stopwords):
+    if lowercase:
+        comment = comment.lower()
+    comment = nlp(comment)
+    lemmatized = list()
+    for word in comment:
+        lemma = word.lemma_.strip()
+        if lemma:
+            if not remove_stopwords or (remove_stopwords and lemma not in stops):
+                lemmatized.append(lemma)
+    return " ".join(lemmatized)
+
+
+# def tokenize_function(examples):
+#     return tokenizer(examples["text"])
+
+
+def compute_metrics(eval_pred):
+    logits, labels = eval_pred
+    predictions = np.argmax(logits, axis=-1)
+    metric = evaluate.load("accuracy")
+    return metric.compute(predictions=predictions, references=labels)
+    
+    
+def get_model():
+    global model_base
+    # last_checkpoint = "./results/checkpoint-22500"
+
+    finetuned_model = T5ForConditionalGeneration.from_pretrained(model_base)
+    tokenizer = T5Tokenizer.from_pretrained(model_base)
+    # model = SentenceTransformer(model_base)
+    gpu_available = torch.cuda.is_available()
+    device = torch.device("cuda" if gpu_available else "cpu")
+    finetuned_model = finetuned_model.to(device)
+    return finetuned_model, tokenizer
+
+
+def cosine_scores(model, sentence):
+    global word1
+    global word2
+    global word3
+    # sentence1 = f"{word1} is to {word2} as"
+    embeddings1 = model.encode(sentence, convert_to_tensor=True)
+
+def embeddings(model, sentences, tokenizer):
+    global word1
+    global word2
+    global word3
+    global model_base
+    gpu_available = torch.cuda.is_available()
+    device = torch.device("cuda" if gpu_available else "cpu")
+    # device = torch.device('cuda:0')
+    # embeddings = model.encode(sentences)
+    question = "Please answer to this question: " + sentences
+    
+    inputs = tokenizer(question, return_tensors="pt")
+    
+    print(inputs)
+    # print(inputs.device)
+    print(model.device)
+    print(inputs['input_ids'].device)
+    print(inputs['attention_mask'].device)
+    
+    inputs['attention_mask'] = inputs['attention_mask'].to(device)
+    inputs['input_ids'] = inputs['input_ids'].to(device)
+    
+    outputs = model.generate(**inputs)
+    answer = tokenizer.decode(outputs[0])
+    answer = answer[6:-4]
+    # print(fill(answer, width=80))
+    
+    print("ANSWER IS", answer)
+
+    return answer
+
+
+def random_word(model, tokenizer):
+    global model_base
+    vocab = tokenizer.get_vocab()
+    # with open(model_base + '/vocab.txt', 'r') as file:
+    line = ""
+    # content = file.readlines()
+    length = tokenizer.vocab_size
+    # print(vocab)
+    while line == "":
+        rand_line = random.randrange(0, length)
+        # print("TRYING TO FIND", rand_line, "OUT OF", length, "WITH VOCAB OF TYPE", type(vocab))
+        for word, id in vocab.items():
+            if id == rand_line and word[0].isalpha() and word not in stops and word not in ROMAN_CONSTANTS:
+        # if vocab[rand_line][0].isalpha() and vocab[rand_line][:-1] not in stops and vocab[rand_line][:-1] not in ROMAN_CONSTANTS:
+                line = word
+            elif id == rand_line:
+                print(f"{word} is not alpha or is a stop word")
+    # for num, aline in enumerate(file, 1997):
+    #     if random.randrange(num) and aline.isalpha():
+    #         continue
+    #     # elif not aline.isalpha():
+            
+    #     line = aline
+    print(line)
+    return line
+
+
+def generate_prompt(model, tokenizer):
+    global word1
+    global word2
+    global word3
+    global answer
+    global base_prompts
+    word1 = random_word(model, tokenizer)
+    # word2 = random_word()
+    
+    word2 = embeddings(model, f"{base_prompts[random.randint(0, len(base_prompts) - 1)]}{word1} is to ___.", tokenizer)
+    word3 = random_word(model, tokenizer)
+    sentence = f"{word1} is to {word2} as {word3} is to ___."
+    print(sentence)
+    answer = embeddings(model, sentence, tokenizer)
+    print("ANSWER IS", answer)
+    return f"# {word1} is to {word2} as {word3} is to ___."
+    # cosine_scores(model, sentence)
+
+
+def greet(name):
+    return "Hello " + name + "!!"
+
+def check_answer(guess:str):
+    global guesses
+    global answer
+    global return_guesses
+    global word1
+    global word2
+    global word3
+    
+    model, tokenizer = get_model()
+    output = ""
+    protected_guess = guess
+    sentence = f"{word1} is to {word2} as [MASK] is to {guess}."
+   
+    other_word = embeddings(model, sentence, tokenizer)
+    guesses.append(guess)
+    
+    
+    
+    for guess in return_guesses:
+        output += ("- " + guess + "<br>")
+    
+    # output = output[:-1]
+    prompt = f"{word1} is to {word2} as {word3} is to ___."
+    # print("IS", protected_guess, "EQUAL TO", answer, ":", protected_guess.lower() == answer.lower())
+    
+    if protected_guess.lower() == answer.lower():
+        return_guesses.append(f"{protected_guess}: {word1} is to {word2} as {word3} is to {protected_guess}.")
+        output += f"<span style='color:green'>- {return_guesses[-1]}</span><br>"
+        new_prompt = generate_prompt(model, tokenizer)
+        return new_prompt, "Correct!", output
+    else:
+        return_guess = f"{protected_guess}: {word1} is to {word2} as {other_word} is to {protected_guess}."
+        return_guesses.append(return_guess)
+        output += ("- " + return_guess + " <br>")
+        return prompt, "Try again!", output
+
+def main():
+    global word1
+    global word2
+    global word3
+    global answer
+    # answer = "Moon"
+    global guesses
+    
+    
+    # num_rows, data_type, value, example, embeddings = training()
+    # sent_embeddings = embeddings()
+    model, tokenizer = get_model() 
+    generate_prompt(model, tokenizer)
+    
+    prompt = f"{word1} is to {word2} as {word3} is to ____"
+    print(prompt)
+    print("TESTING EMBEDDINGS")
+    with gr.Blocks() as iface:
+        mark_question = gr.Markdown(prompt)
+        with gr.Tab("Guess"):
+            text_input = gr.Textbox()
+            text_output = gr.Textbox()
+            text_button = gr.Button("Submit")
+        with gr.Accordion("Open for previous guesses"):
+            text_guesses = gr.Markdown()
+        # with gr.Tab("Testing"):
+        #     gr.Markdown(f"""The Embeddings are {sent_embeddings}.""")
+        text_button.click(check_answer, inputs=[text_input], outputs=[mark_question, text_output, text_guesses])
+    # iface = gr.Interface(fn=greet, inputs="text", outputs="text")
+    iface.launch()
+    
+    
+
+
+    
+if __name__ == "__main__":
     main()

flan-t5-train.py

@@ -1,302 +1,235 @@
-import gradio as gr
-import math
-from datasets import load_dataset
-from transformers import AutoTokenizer, AutoModel, AutoModelForSequenceClassification
-from transformers import TrainingArguments, Trainer
-from transformers import T5Tokenizer, T5ForConditionalGeneration
-import torch
-import torch.nn.functional as F
-from torch.utils.data import DataLoader
-import numpy as np
-import evaluate
-import nltk
-from nltk.corpus import stopwords
-import subprocess
-import sys
-from transformers import T5Tokenizer, DataCollatorForSeq2Seq
-from transformers import T5ForConditionalGeneration, Seq2SeqTrainingArguments, Seq2SeqTrainer
-from transformers import DataCollatorWithPadding, DistilBertTokenizerFast
-from transformers import TrainingArguments
-from transformers import (
-    BertModel,
-    BertTokenizerFast,
-    Trainer,
-    EvalPrediction
-)
-
-# !pip install https://huggingface.co/spacy/en_core_web_sm/resolve/main/en_core_web_sm-any-py3-none-any.whl
-# subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'https://huggingface.co/spacy/en_core_web_sm/resolve/main/en_core_web_sm-any-py3-none-any.whl'])
-# tokenizer = AutoTokenizer.from_pretrained('sentence-transformers/all-MiniLM-L6-v2')
-# data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
-# nltk.download('stopwords')
-# nlp = spacy.load("en_core_web_sm")
-# stops = stopwords.words("english")
-nltk.download("punkt", quiet=True)
-metric = evaluate.load("rouge")
-
-# Global Parameters
-L_RATE = 3e-4
-BATCH_SIZE = 8
-PER_DEVICE_EVAL_BATCH = 4
-WEIGHT_DECAY = 0.01
-SAVE_TOTAL_LIM = 3
-NUM_EPOCHS = 10
-
-# Set up training arguments
-training_args = Seq2SeqTrainingArguments(
-   output_dir="./results",
-   evaluation_strategy="epoch",
-   learning_rate=L_RATE,
-   per_device_train_batch_size=BATCH_SIZE,
-   per_device_eval_batch_size=PER_DEVICE_EVAL_BATCH,
-   weight_decay=WEIGHT_DECAY,
-   save_total_limit=SAVE_TOTAL_LIM,
-   num_train_epochs=NUM_EPOCHS,
-   predict_with_generate=True,
-   push_to_hub=False
-)
-
-model_id = "google/flan-t5-base"
-tokenizer = T5Tokenizer.from_pretrained(model_id)
-# tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')
-# metric = evaluate.load("accuracy")
-
-def tokenize_function(examples):
-    return tokenizer(examples["stem"], padding="max_length", truncation=True)
-
-
-#Mean Pooling - Take attention mask into account for correct averaging
-def mean_pooling(model_output, attention_mask):
-    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
-    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
-    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
-
-
-# def compute_metrics(eval_pred):
-#     logits, labels = eval_pred
-#     predictions = np.argmax(logits, axis=-1)
-#     metric = evaluate.load("accuracy")
-#     return metric.compute(predictions=predictions, references=labels)
-
-def compute_metrics(eval_preds):
-   preds, labels = eval_preds
-
-   # decode preds and labels
-   labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
-   decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)
-   decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
-
-   # rougeLSum expects newline after each sentence
-   decoded_preds = ["\n".join(nltk.sent_tokenize(pred.strip())) for pred in decoded_preds]
-   decoded_labels = ["\n".join(nltk.sent_tokenize(label.strip())) for label in decoded_labels]
-
-   result = metric.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True)
-  
-   return result
-
-
-def training():
-    dataset_id = "tomasmcz/word2vec_analogy"
-    # dataset_id = "relbert/scientific_and_creative_analogy"
-    # dataset_sub = "Quadruples_Kmiecik_random_split"
-    print("GETTING DATASET")
-    dataset = load_dataset(dataset_id)
-    # dataset = dataset["train"]
-    # tokenized_datasets = dataset.map(tokenize_function, batched=True)
-    
-    print(dataset)
-    print(f"- The {dataset_id} dataset has {dataset['train'].num_rows} examples.")
-    print(f"- Each example is a {type(dataset['train'][0])} with a {type(dataset['train'][0])} as value.")
-    print(f"- Examples look like this: {dataset['train'][0]}")
-    
-    # for i in dataset["train"]:
-    #     print(i["AB"], "to", i["CD"], "is", i["label"])
-    
-    dataset = dataset["train"].train_test_split(test_size=0.3)
-    
-    # We prefix our tasks with "answer the question"
-    prefix = "Please answer this question: "
-
-    # Define the preprocessing function
-
-    # def preprocess_function(examples):
-    #     """Add prefix to the sentences, tokenize the text, and set the labels"""
-    #     # The "inputs" are the tokenized answer:
-    #     inputs = []
-    #     # print(examples)
-    #     # inputs = [prefix + doc for doc in examples["question"]]
-    #     for doc in examples['source']:
-    #         # print("THE DOC IS:", doc)
-    #         # print("THE DOC IS:", examples[i]['AB'], examples[i]['CD'], examples[i]['label'])
-    #         prompt = f"{prefix}map "
-    #         for item in doc:
-    #             prompt += f"{item}, and "
-    #         prompt = prompt[:-6]
-    #         inputs.append(prompt)
-    #     # inputs = [prefix + doc for doc in examples["question"]]
-    #     for indx, doc in enumerate(examples["target_random"]):
-    #         prompt = f" to "
-    #         for item in doc:
-    #             prompt += f"{item}, and "
-    #         prompt = prompt[:-6] + "."
-    #         inputs[indx] += prompt
-    #     model_inputs = tokenizer(inputs, max_length=128, truncation=True)
-        
-    def preprocess_function(examples):
-        """Add prefix to the sentences, tokenize the text, and set the labels"""
-        # The "inputs" are the tokenized answer:
-        inputs = []
-        # print(examples)
-        # inputs = [prefix + doc for doc in examples["question"]]
-        for doc in examples['word_a']:
-            # print("THE DOC IS:", doc)
-            # print("THE DOC IS:", examples[i]['AB'], examples[i]['CD'], examples[i]['label'])
-            prompt = f"{prefix}{doc} is to "
-            inputs.append(prompt)
-        # inputs = [prefix + doc for doc in examples["question"]]
-        for indx, doc in enumerate(examples["word_b"]):
-            prompt = f"{doc} as "
-            inputs[indx] += prompt
-            
-        for indx, doc in enumerate(examples["word_c"]):
-            prompt = f"{doc} is to ___."
-            inputs[indx] += prompt
-        model_inputs = tokenizer(inputs, max_length=128, truncation=True)
-        
-        # print(examples["label"], type(examples["label"]))
-        
-        # The "labels" are the tokenized outputs:
-        labels = tokenizer(text_target=examples["word_d"], 
-                            max_length=512,         
-                            truncation=True)
-
-        model_inputs["labels"] = labels["input_ids"]
-        return model_inputs
-    
-    
-    
-    # Map the preprocessing function across our dataset
-    tokenized_dataset = dataset.map(preprocess_function, batched=True)
-    # train_examples = []
-    # train_data = dataset["test"]
-    # # For agility we only 1/2 of our available data
-    # n_examples = dataset["test"].num_rows // 2
-    
-    # for i in range(n_examples):
-    #     example = train_data[i]
-    #     temp_word_1 = example["stem"][0]
-    #     temp_word_2 = example["stem"][1]
-    #     temp_word_3 = example["choice"][example["answer"]][0]
-    #     temp_word_4 = example["choice"][example["answer"]][1]
-    #     comp1 = f"{temp_word_1} to {temp_word_2}"
-    #     comp2 = f"{temp_word_3} to {temp_word_4}"
-    #     # example_opposite = dataset_clean[-(i)]
-    #     # print(example["text"])
-    #     train_examples.append(InputExample(texts=[comp1, comp2]))
-    
-        
-    # train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=25)
-    
-    print("END DATALOADER")
-    
-    # print(train_examples)
-        
-    embeddings = finetune(tokenized_dataset)
-    
-    return 0
-
-
-def finetune(dataset):
-    # model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5)
-    # model_id = "sentence-transformers/all-MiniLM-L6-v2"
-    model_id = "google/flan-t5-base"
-    # model_id = "distilbert-base-uncased"
-    # tokenizer = DistilBertTokenizerFast.from_pretrained(model_id)
-    tokenizer = T5Tokenizer.from_pretrained(model_id)
-    model = T5ForConditionalGeneration.from_pretrained(model_id)
-    data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)
-    device = torch.device('cuda:0')
-    model = model.to(device)
-    
-    # training_args = TrainingArguments(output_dir="test_trainer")
-    
-    # USE THIS LINK
-    # https://huggingface.co/blog/how-to-train-sentence-transformers
-    
-    # train_loss = losses.MegaBatchMarginLoss(model=model)
-    # ds_train, ds_valid = dataset.train_test_split(test_size=0.2, seed=42)
-    
-    print("BEGIN FIT")
-    
-    trainer = Seq2SeqTrainer(
-        model=model,
-        args=training_args,
-        train_dataset=dataset["train"],
-        eval_dataset=dataset["test"],
-        # evaluation_strategy="no"
-        tokenizer=tokenizer,
-        data_collator=data_collator,
-        compute_metrics=compute_metrics
-        )
-    
-    # model.fit(train_objectives=[(train_dataloader, train_loss)], epochs=10)
-    
-    trainer.train()
-    
-    # model.save("flan-analogies")
-    
-    # model.save_to_hub("smhavens/bert-base-analogies")
-    # accuracy = compute_metrics(eval, metric)
-    return 0
-
-def greet(name):
-    return "Hello " + name + "!!"
-
-def check_answer(guess:str):
-    global guesses
-    global answer
-    guesses.append(guess)
-    output = ""
-    for guess in guesses:
-        output += ("- " + guess + "\n")
-    output = output[:-1]
-    
-    if guess.lower() == answer.lower():
-        return "Correct!", output
-    else:
-        return "Try again!", output
-
-def main():
-    print("BEGIN")
-    word1 = "Black"
-    word2 = "White"
-    word3 = "Sun"
-    global answer
-    answer = "Moon"
-    global guesses
-    
-    training()
-    
-    # prompt = f"{word1} is to {word2} as {word3} is to ____"
-    # with gr.Blocks() as iface:
-    #     gr.Markdown(prompt)
-    #     with gr.Tab("Guess"):
-    #         text_input = gr.Textbox()
-    #         text_output = gr.Textbox()
-    #         text_button = gr.Button("Submit")
-    #     with gr.Accordion("Open for previous guesses"):
-    #         text_guesses = gr.Textbox()
-    #     with gr.Tab("Testing"):
-    #         gr.Markdown(f"""Number of rows in dataset is {num_rows}, with each having type {data_type} and value {value}.
-    #                     An example is {example}.
-    #                     The Embeddings are {embeddings}.""")
-    #     text_button.click(check_answer, inputs=[text_input], outputs=[text_output, text_guesses])
-    # # iface = gr.Interface(fn=greet, inputs="text", outputs="text")
-    # iface.launch()
-    
-    
-
-
-    
-if __name__ == "__main__":
+import gradio as gr
+import math
+from datasets import load_dataset
+from transformers import AutoTokenizer, AutoModel, AutoModelForSequenceClassification
+from transformers import TrainingArguments, Trainer
+from transformers import T5Tokenizer, T5ForConditionalGeneration
+import torch
+import torch.nn.functional as F
+from torch.utils.data import DataLoader
+import numpy as np
+import evaluate
+import nltk
+from nltk.corpus import stopwords
+import subprocess
+import sys
+from transformers import T5Tokenizer, DataCollatorForSeq2Seq
+from transformers import T5ForConditionalGeneration, Seq2SeqTrainingArguments, Seq2SeqTrainer
+from transformers import DataCollatorWithPadding, DistilBertTokenizerFast
+from transformers import TrainingArguments
+from transformers import (
+    BertModel,
+    BertTokenizerFast,
+    Trainer,
+    EvalPrediction
+)
+
+nltk.download("punkt", quiet=True)
+metric = evaluate.load("rouge")
+
+# Global Parameters
+L_RATE = 3e-4
+BATCH_SIZE = 8
+PER_DEVICE_EVAL_BATCH = 4
+WEIGHT_DECAY = 0.01
+SAVE_TOTAL_LIM = 3
+NUM_EPOCHS = 10
+
+# Set up training arguments
+training_args = Seq2SeqTrainingArguments(
+   output_dir="./results",
+   evaluation_strategy="epoch",
+   learning_rate=L_RATE,
+   per_device_train_batch_size=BATCH_SIZE,
+   per_device_eval_batch_size=PER_DEVICE_EVAL_BATCH,
+   weight_decay=WEIGHT_DECAY,
+   save_total_limit=SAVE_TOTAL_LIM,
+   num_train_epochs=NUM_EPOCHS,
+   predict_with_generate=True,
+   push_to_hub=False
+)
+
+model_id = "google/flan-t5-base"
+tokenizer = T5Tokenizer.from_pretrained(model_id)
+# tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')
+# metric = evaluate.load("accuracy")
+
+def tokenize_function(examples):
+    return tokenizer(examples["stem"], padding="max_length", truncation=True)
+
+
+#Mean Pooling - Take attention mask into account for correct averaging
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+
+# def compute_metrics(eval_pred):
+#     logits, labels = eval_pred
+#     predictions = np.argmax(logits, axis=-1)
+#     metric = evaluate.load("accuracy")
+#     return metric.compute(predictions=predictions, references=labels)
+
+def compute_metrics(eval_preds):
+   preds, labels = eval_preds
+
+   # decode preds and labels
+   labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
+   decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)
+   decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
+
+   # rougeLSum expects newline after each sentence
+   decoded_preds = ["\n".join(nltk.sent_tokenize(pred.strip())) for pred in decoded_preds]
+   decoded_labels = ["\n".join(nltk.sent_tokenize(label.strip())) for label in decoded_labels]
+
+   result = metric.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True)
+  
+   return result
+
+
+def training():
+    dataset_id = "tomasmcz/word2vec_analogy"
+    # dataset_id = "relbert/scientific_and_creative_analogy"
+    # dataset_sub = "Quadruples_Kmiecik_random_split"
+    print("GETTING DATASET")
+    dataset = load_dataset(dataset_id)
+    # dataset = dataset["train"]
+    # tokenized_datasets = dataset.map(tokenize_function, batched=True)
+    
+    print(dataset)
+    print(f"- The {dataset_id} dataset has {dataset['train'].num_rows} examples.")
+    print(f"- Each example is a {type(dataset['train'][0])} with a {type(dataset['train'][0])} as value.")
+    print(f"- Examples look like this: {dataset['train'][0]}")
+    
+    # for i in dataset["train"]:
+    #     print(i["AB"], "to", i["CD"], "is", i["label"])
+    
+    dataset = dataset["train"].train_test_split(test_size=0.3)
+    
+    # We prefix our tasks with "answer the question"
+    prefix = "Please answer this question: "
+
+        
+    def preprocess_function(examples):
+        """Add prefix to the sentences, tokenize the text, and set the labels"""
+        # The "inputs" are the tokenized answer:
+        inputs = []
+        # print(examples)
+        # inputs = [prefix + doc for doc in examples["question"]]
+        for doc in examples['word_a']:
+            # print("THE DOC IS:", doc)
+            # print("THE DOC IS:", examples[i]['AB'], examples[i]['CD'], examples[i]['label'])
+            prompt = f"{prefix}{doc} is to "
+            inputs.append(prompt)
+        # inputs = [prefix + doc for doc in examples["question"]]
+        for indx, doc in enumerate(examples["word_b"]):
+            prompt = f"{doc} as "
+            inputs[indx] += prompt
+            
+        for indx, doc in enumerate(examples["word_c"]):
+            prompt = f"{doc} is to ___."
+            inputs[indx] += prompt
+        model_inputs = tokenizer(inputs, max_length=128, truncation=True)
+        
+        # print(examples["label"], type(examples["label"]))
+        
+        # The "labels" are the tokenized outputs:
+        labels = tokenizer(text_target=examples["word_d"], 
+                            max_length=512,         
+                            truncation=True)
+
+        model_inputs["labels"] = labels["input_ids"]
+        return model_inputs
+    
+    
+    
+    # Map the preprocessing function across our dataset
+    tokenized_dataset = dataset.map(preprocess_function, batched=True)
+    
+    print("END DATALOADER")
+    
+    # print(train_examples)
+        
+    embeddings = finetune(tokenized_dataset)
+    
+    return 0
+
+
+def finetune(dataset):
+    # model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5)
+    # model_id = "sentence-transformers/all-MiniLM-L6-v2"
+    model_id = "google/flan-t5-base"
+    # model_id = "distilbert-base-uncased"
+    # tokenizer = DistilBertTokenizerFast.from_pretrained(model_id)
+    tokenizer = T5Tokenizer.from_pretrained(model_id)
+    model = T5ForConditionalGeneration.from_pretrained(model_id)
+    data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)
+    device = torch.device('cuda:0')
+    model = model.to(device)
+    
+    # training_args = TrainingArguments(output_dir="test_trainer")
+    
+    # USE THIS LINK
+    # https://huggingface.co/blog/how-to-train-sentence-transformers
+    
+    # train_loss = losses.MegaBatchMarginLoss(model=model)
+    # ds_train, ds_valid = dataset.train_test_split(test_size=0.2, seed=42)
+    
+    print("BEGIN FIT")
+    
+    trainer = Seq2SeqTrainer(
+        model=model,
+        args=training_args,
+        train_dataset=dataset["train"],
+        eval_dataset=dataset["test"],
+        # evaluation_strategy="no"
+        tokenizer=tokenizer,
+        data_collator=data_collator,
+        compute_metrics=compute_metrics
+        )
+    
+    # model.fit(train_objectives=[(train_dataloader, train_loss)], epochs=10)
+    
+    trainer.train()
+    
+    # model.save("flan-analogies")
+    
+    # model.save_to_hub("smhavens/bert-base-analogies")
+    # accuracy = compute_metrics(eval, metric)
+    return 0
+
+def greet(name):
+    return "Hello " + name + "!!"
+
+def check_answer(guess:str):
+    global guesses
+    global answer
+    guesses.append(guess)
+    output = ""
+    for guess in guesses:
+        output += ("- " + guess + "\n")
+    output = output[:-1]
+    
+    if guess.lower() == answer.lower():
+        return "Correct!", output
+    else:
+        return "Try again!", output
+
+def main():
+    print("BEGIN")
+    word1 = "Black"
+    word2 = "White"
+    word3 = "Sun"
+    global answer
+    answer = "Moon"
+    global guesses
+    
+    training()
+    
+    
+
+
+    
+if __name__ == "__main__":
     main()

results/checkpoint-16000/added_tokens.json

@@ -0,0 +1,102 @@
+{
+  "<extra_id_0>": 32099,
+  "<extra_id_10>": 32089,
+  "<extra_id_11>": 32088,
+  "<extra_id_12>": 32087,
+  "<extra_id_13>": 32086,
+  "<extra_id_14>": 32085,
+  "<extra_id_15>": 32084,
+  "<extra_id_16>": 32083,
+  "<extra_id_17>": 32082,
+  "<extra_id_18>": 32081,
+  "<extra_id_19>": 32080,
+  "<extra_id_1>": 32098,
+  "<extra_id_20>": 32079,
+  "<extra_id_21>": 32078,
+  "<extra_id_22>": 32077,
+  "<extra_id_23>": 32076,
+  "<extra_id_24>": 32075,
+  "<extra_id_25>": 32074,
+  "<extra_id_26>": 32073,
+  "<extra_id_27>": 32072,
+  "<extra_id_28>": 32071,
+  "<extra_id_29>": 32070,
+  "<extra_id_2>": 32097,
+  "<extra_id_30>": 32069,
+  "<extra_id_31>": 32068,
+  "<extra_id_32>": 32067,
+  "<extra_id_33>": 32066,
+  "<extra_id_34>": 32065,
+  "<extra_id_35>": 32064,
+  "<extra_id_36>": 32063,
+  "<extra_id_37>": 32062,
+  "<extra_id_38>": 32061,
+  "<extra_id_39>": 32060,
+  "<extra_id_3>": 32096,
+  "<extra_id_40>": 32059,
+  "<extra_id_41>": 32058,
+  "<extra_id_42>": 32057,
+  "<extra_id_43>": 32056,
+  "<extra_id_44>": 32055,
+  "<extra_id_45>": 32054,
+  "<extra_id_46>": 32053,
+  "<extra_id_47>": 32052,
+  "<extra_id_48>": 32051,
+  "<extra_id_49>": 32050,
+  "<extra_id_4>": 32095,
+  "<extra_id_50>": 32049,
+  "<extra_id_51>": 32048,
+  "<extra_id_52>": 32047,
+  "<extra_id_53>": 32046,
+  "<extra_id_54>": 32045,
+  "<extra_id_55>": 32044,
+  "<extra_id_56>": 32043,
+  "<extra_id_57>": 32042,
+  "<extra_id_58>": 32041,
+  "<extra_id_59>": 32040,
+  "<extra_id_5>": 32094,
+  "<extra_id_60>": 32039,
+  "<extra_id_61>": 32038,
+  "<extra_id_62>": 32037,
+  "<extra_id_63>": 32036,
+  "<extra_id_64>": 32035,
+  "<extra_id_65>": 32034,
+  "<extra_id_66>": 32033,
+  "<extra_id_67>": 32032,
+  "<extra_id_68>": 32031,
+  "<extra_id_69>": 32030,
+  "<extra_id_6>": 32093,
+  "<extra_id_70>": 32029,
+  "<extra_id_71>": 32028,
+  "<extra_id_72>": 32027,
+  "<extra_id_73>": 32026,
+  "<extra_id_74>": 32025,
+  "<extra_id_75>": 32024,
+  "<extra_id_76>": 32023,
+  "<extra_id_77>": 32022,
+  "<extra_id_78>": 32021,
+  "<extra_id_79>": 32020,
+  "<extra_id_7>": 32092,
+  "<extra_id_80>": 32019,
+  "<extra_id_81>": 32018,
+  "<extra_id_82>": 32017,
+  "<extra_id_83>": 32016,
+  "<extra_id_84>": 32015,
+  "<extra_id_85>": 32014,
+  "<extra_id_86>": 32013,
+  "<extra_id_87>": 32012,
+  "<extra_id_88>": 32011,
+  "<extra_id_89>": 32010,
+  "<extra_id_8>": 32091,
+  "<extra_id_90>": 32009,
+  "<extra_id_91>": 32008,
+  "<extra_id_92>": 32007,
+  "<extra_id_93>": 32006,
+  "<extra_id_94>": 32005,
+  "<extra_id_95>": 32004,
+  "<extra_id_96>": 32003,
+  "<extra_id_97>": 32002,
+  "<extra_id_98>": 32001,
+  "<extra_id_99>": 32000,
+  "<extra_id_9>": 32090
+}

results/checkpoint-16000/config.json

@@ -0,0 +1,62 @@
+{
+  "_name_or_path": "google/flan-t5-base",
+  "architectures": [
+    "T5ForConditionalGeneration"
+  ],
+  "classifier_dropout": 0.0,
+  "d_ff": 2048,
+  "d_kv": 64,
+  "d_model": 768,
+  "decoder_start_token_id": 0,
+  "dense_act_fn": "gelu_new",
+  "dropout_rate": 0.1,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": true,
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "t5",
+  "n_positions": 512,
+  "num_decoder_layers": 12,
+  "num_heads": 12,
+  "num_layers": 12,
+  "output_past": true,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "task_specific_params": {
+    "summarization": {
+      "early_stopping": true,
+      "length_penalty": 2.0,
+      "max_length": 200,
+      "min_length": 30,
+      "no_repeat_ngram_size": 3,
+      "num_beams": 4,
+      "prefix": "summarize: "
+    },
+    "translation_en_to_de": {
+      "early_stopping": true,
+      "max_length": 300,
+      "num_beams": 4,
+      "prefix": "translate English to German: "
+    },
+    "translation_en_to_fr": {
+      "early_stopping": true,
+      "max_length": 300,
+      "num_beams": 4,
+      "prefix": "translate English to French: "
+    },
+    "translation_en_to_ro": {
+      "early_stopping": true,
+      "max_length": 300,
+      "num_beams": 4,
+      "prefix": "translate English to Romanian: "
+    }
+  },
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.2",
+  "use_cache": true,
+  "vocab_size": 32128
+}

results/checkpoint-16000/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "decoder_start_token_id": 0,
+  "eos_token_id": 1,
+  "pad_token_id": 0,
+  "transformers_version": "4.35.2"
+}

results/checkpoint-16000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd7f96db75733e18d6af8488ab51eea991be641c6c22b24fa5ab3b45101c3398
+size 990345064

results/checkpoint-16000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31aa07bcfc63b03b9dbfb77536457e4d0591b64d537e2f4834f5b81c6bd2ab21
+size 1980860410

results/checkpoint-16000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc296e1811c88d4548bfa74b8cf96485e58c41652ba8a0db69b6e3a9762f9be0
+size 14244

results/checkpoint-16000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c77d751bb87ca04afd8f823ee9102cffea6221900b1a056c2f31d9044f1a0ce
+size 1064

results/checkpoint-16000/special_tokens_map.json

@@ -0,0 +1,125 @@
+{
+  "additional_special_tokens": [
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>"
+  ],
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

results/checkpoint-16000/spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d60acb128cf7b7f2536e8f38a5b18a05535c9e14c7a355904270e15b0945ea86
+size 791656