still broken?

analogy_train.py

@@ -1,301 +1,301 @@
-import gradio as gr
-import math
-import spacy
-from datasets import load_dataset
-from sentence_transformers import SentenceTransformer
-from sentence_transformers import InputExample
-from sentence_transformers import losses
-from transformers import AutoTokenizer, AutoModel, AutoModelForSequenceClassification
-from transformers import TrainingArguments, Trainer
-import torch
-import torch.nn.functional as F
-from torch.utils.data import DataLoader
-import numpy as np
-import evaluate
-import nltk
-from nltk.corpus import stopwords
-import subprocess
-import sys
-from transformers import DataCollatorWithPadding
-from transformers import TrainingArguments
-from transformers import (
-    BertModel,
-    BertTokenizerFast,
-    Trainer,
-    EvalPrediction
-)
-
-# !pip install https://huggingface.co/spacy/en_core_web_sm/resolve/main/en_core_web_sm-any-py3-none-any.whl
-# subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'https://huggingface.co/spacy/en_core_web_sm/resolve/main/en_core_web_sm-any-py3-none-any.whl'])
-# tokenizer = AutoTokenizer.from_pretrained('sentence-transformers/all-MiniLM-L6-v2')
-# data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
-# nltk.download('stopwords')
-# nlp = spacy.load("en_core_web_sm")
-# stops = stopwords.words("english")
-
-# answer = "Pizza"
-guesses = []
-answer = "Pizza"
-
-tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')
-metric = evaluate.load("accuracy")
-
-def tokenize_function(examples):
-    return tokenizer(examples["stem"], padding="max_length", truncation=True)
-
-
-#Mean Pooling - Take attention mask into account for correct averaging
-def mean_pooling(model_output, attention_mask):
-    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
-    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
-    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
-
-
-def compute_metrics(eval_pred):
-    logits, labels = eval_pred
-    predictions = np.argmax(logits, axis=-1)
-    metric = evaluate.load("accuracy")
-    return metric.compute(predictions=predictions, references=labels)
-
-
-# def training():
-#     dataset_id = "relbert/analogy_questions"
-#     dataset_sub = "bats"
-#     print("GETTING DATASET")
-#     raw_dataset = load_dataset(dataset_id, dataset_sub)
-#     # data_metric = evaluate.load(dataset_id, dataset_sub)
-#     checkpoint = "bert-base-uncased"
-#     model = BertModel.from_pretrained(checkpoint)
-#     # dataset = dataset["train"]
-#     # tokenized_datasets = dataset.map(tokenize_function, batched=True)
-#     # print(raw_dataset)
-#     test_data = raw_dataset["test"]
-#     # print(test_data["stem"])
-#     all_answers = []
-#     for answer in raw_dataset["answer"]:
-#         answer = raw_dataset["choice"][answer]
-#     raw_dataset = raw_dataset.add_column("label", all_answers)
-    
-        
-#     print(raw_dataset)
-#     print(raw_dataset["label"])
-#     dataset = raw_dataset.map(
-#         lambda x: tokenizer(x["stem"], truncation=True),
-#         batched=True,
-#     )
-#     print(dataset)
-#     dataset = dataset.remove_columns(["stem", "answer", "choice"])
-#     dataset = dataset.rename_column("label", "labels")
-#     dataset = dataset.with_format("torch")
-
-#     training_args = TrainingArguments("test-trainer", evaluation_strategy="epoch")
-
-#     print(dataset)
-#     # print(f"- The {dataset_id} dataset has {dataset.num_rows} examples.")
-#     # print(f"- Each example is a {type(dataset[0])} with a {type(dataset[0]['stem'])} as value.")
-#     # print(f"- Examples look like this: {dataset[0]}")
-    
-#     # small_train_dataset = tokenized_datasets["train"].shuffle(seed=42).select(range(1000))
-#     # small_eval_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(1000))
-    
-#     # dataset = dataset["train"].map(tokenize_function, batched=True)
-#     # dataset.set_format(type="torch", columns=["input_ids", "token_type_ids", "attention_mask", "label"])
-#     # dataset.format['type']
-    
-#     # tokenized_news = dataset.map(tokenize_function, batched=True)
-    
-#     # model = AutoModelForSequenceClassification.from_pretrained("sentence-transformers/all-MiniLM-L6-v2", num_labels=2)
-    
-#     # print(dataset)
-    
-#     # Choose the appropriate device based on availability (CUDA or CPU)
-#     # gpu_available = torch.cuda.is_available()
-#     # device = torch.device("cuda" if gpu_available else "cpu")
-#     # model = AutoModelForSequenceClassification.from_pretrained('bert-base-uncased')
-    
-#     # tokenized_datasets = dataset.map(tokenize_function, batched=True)
-#     # print(tokenized_datasets)
-#     # # small_train_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(1000))
-#     # # small_eval_dataset = tokenized_datasets["validation"].shuffle(seed=42).select(range(1000))
-    
-#     # model = model.to(device)
-    
-#     # model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5)
-#     # training_args = TrainingArguments(output_dir="test_trainer")
-    
-#     trainer = Trainer(
-#         model=model,
-#         args=training_args,
-#         train_dataset=dataset["test"],
-#         eval_dataset=dataset["validation"],
-#         compute_metrics=compute_metrics,
-#     )
-    
-#     output = trainer.train()
-    
-#     # train_examples = []
-#     # train_data = dataset["train"]
-#     # # For agility we only 1/2 of our available data
-#     # n_examples = dataset["train"].num_rows // 2
-    
-#     # for i in range(n_examples):
-#     #     example = train_data[i]
-#     #     # example_opposite = dataset_clean[-(i)]
-#     #     # print(example["text"])
-#     #     train_examples.append(InputExample(texts=[example['stem'], example]))
-    
-        
-#     # train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=25)
-    
-#     # print("END DATALOADER")
-    
-#     # # print(train_examples)
-        
-#     # embeddings = finetune(train_dataloader)
-#     print(output)
-    
-#     model.save("bert-analogies")
-    
-#     model.save_to_hub("smhavens/bert-base-analogies")
-#     return output
-
-
-# def finetune(train_dataloader):
-#     # model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5)
-#     model_id = "sentence-transformers/all-MiniLM-L6-v2"
-#     model = SentenceTransformer(model_id)
-#     device = torch.device('cuda:0')
-#     model = model.to(device)
-    
-#     # training_args = TrainingArguments(output_dir="test_trainer")
-    
-#     # USE THIS LINK
-#     # https://huggingface.co/blog/how-to-train-sentence-transformers
-    
-#     train_loss = losses.BatchHardSoftMarginTripletLoss(model=model)
-    
-#     print("BEGIN FIT")
-    
-#     model.fit(train_objectives=[(train_dataloader, train_loss)], epochs=10)
-    
-#     model.save("bert-analogies")
-    
-#     model.save_to_hub("smhavens/bert-base-analogies")
-#     return 0
-
-def training():
-    dataset_id = "relbert/analogy_questions"
-    dataset_sub = "bats"
-    print("GETTING DATASET")
-    dataset = load_dataset(dataset_id, dataset_sub)
-    # dataset = dataset["train"]
-    # tokenized_datasets = dataset.map(tokenize_function, batched=True)
-    
-    print(f"- The {dataset_id} dataset has {dataset['test'].num_rows} examples.")
-    print(f"- Each example is a {type(dataset['test'][0])} with a {type(dataset['test'][0]['stem'])} as value.")
-    print(f"- Examples look like this: {dataset['test'][0]}")
-    
-    train_examples = []
-    train_data = dataset["test"]
-    # For agility we only 1/2 of our available data
-    n_examples = dataset["test"].num_rows // 2
-    
-    for i in range(n_examples):
-        example = train_data[i]
-        temp_word_1 = example["stem"][0]
-        temp_word_2 = example["stem"][1]
-        temp_word_3 = example["choice"][example["answer"]][0]
-        temp_word_4 = example["choice"][example["answer"]][1]
-        comp1 = f"{temp_word_1} to {temp_word_2}"
-        comp2 = f"{temp_word_3} to {temp_word_4}"
-        # example_opposite = dataset_clean[-(i)]
-        # print(example["text"])
-        train_examples.append(InputExample(texts=[comp1, comp2]))
-    
-        
-    train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=25)
-    
-    print("END DATALOADER")
-    
-    # print(train_examples)
-        
-    embeddings = finetune(train_dataloader)
-    
-    return (dataset['test'].num_rows, type(dataset['test'][0]), type(dataset['test'][0]['stem']), dataset['test'][0], embeddings)
-
-
-def finetune(train_dataloader):
-    # model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5)
-    model_id = "sentence-transformers/all-MiniLM-L6-v2"
-    model = SentenceTransformer(model_id)
-    device = torch.device('cuda:0')
-    model = model.to(device)
-    
-    # training_args = TrainingArguments(output_dir="test_trainer")
-    
-    # USE THIS LINK
-    # https://huggingface.co/blog/how-to-train-sentence-transformers
-    
-    train_loss = losses.MegaBatchMarginLoss(model=model)
-    
-    print("BEGIN FIT")
-    
-    model.fit(train_objectives=[(train_dataloader, train_loss)], epochs=10)
-    
-    model.save("bert-analogies")
-    
-    # model.save_to_hub("smhavens/bert-base-analogies")
-    # accuracy = compute_metrics(eval, metric)
-    return 0
-
-def greet(name):
-    return "Hello " + name + "!!"
-
-def check_answer(guess:str):
-    global guesses
-    global answer
-    guesses.append(guess)
-    output = ""
-    for guess in guesses:
-        output += ("- " + guess + "\n")
-    output = output[:-1]
-    
-    if guess.lower() == answer.lower():
-        return "Correct!", output
-    else:
-        return "Try again!", output
-
-def main():
-    print("BEGIN")
-    word1 = "Black"
-    word2 = "White"
-    word3 = "Sun"
-    global answer
-    answer = "Moon"
-    global guesses
-    
-    num_rows, data_type, value, example, embeddings = training()
-    
-    # prompt = f"{word1} is to {word2} as {word3} is to ____"
-    # with gr.Blocks() as iface:
-    #     gr.Markdown(prompt)
-    #     with gr.Tab("Guess"):
-    #         text_input = gr.Textbox()
-    #         text_output = gr.Textbox()
-    #         text_button = gr.Button("Submit")
-    #     with gr.Accordion("Open for previous guesses"):
-    #         text_guesses = gr.Textbox()
-    #     with gr.Tab("Testing"):
-    #         gr.Markdown(f"""Number of rows in dataset is {num_rows}, with each having type {data_type} and value {value}.
-    #                     An example is {example}.
-    #                     The Embeddings are {embeddings}.""")
-    #     text_button.click(check_answer, inputs=[text_input], outputs=[text_output, text_guesses])
-    # # iface = gr.Interface(fn=greet, inputs="text", outputs="text")
-    # iface.launch()
-    
-    
-
-
-    
-if __name__ == "__main__":
+import gradio as gr
+import math
+import spacy
+from datasets import load_dataset
+from sentence_transformers import SentenceTransformer
+from sentence_transformers import InputExample
+from sentence_transformers import losses
+from transformers import AutoTokenizer, AutoModel, AutoModelForSequenceClassification
+from transformers import TrainingArguments, Trainer
+import torch
+import torch.nn.functional as F
+from torch.utils.data import DataLoader
+import numpy as np
+import evaluate
+import nltk
+from nltk.corpus import stopwords
+import subprocess
+import sys
+from transformers import DataCollatorWithPadding
+from transformers import TrainingArguments
+from transformers import (
+    BertModel,
+    BertTokenizerFast,
+    Trainer,
+    EvalPrediction
+)
+
+# !pip install https://huggingface.co/spacy/en_core_web_sm/resolve/main/en_core_web_sm-any-py3-none-any.whl
+# subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'https://huggingface.co/spacy/en_core_web_sm/resolve/main/en_core_web_sm-any-py3-none-any.whl'])
+# tokenizer = AutoTokenizer.from_pretrained('sentence-transformers/all-MiniLM-L6-v2')
+# data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
+# nltk.download('stopwords')
+# nlp = spacy.load("en_core_web_sm")
+# stops = stopwords.words("english")
+
+# answer = "Pizza"
+guesses = []
+answer = "Pizza"
+
+tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')
+metric = evaluate.load("accuracy")
+
+def tokenize_function(examples):
+    return tokenizer(examples["stem"], padding="max_length", truncation=True)
+
+
+#Mean Pooling - Take attention mask into account for correct averaging
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+
+def compute_metrics(eval_pred):
+    logits, labels = eval_pred
+    predictions = np.argmax(logits, axis=-1)
+    metric = evaluate.load("accuracy")
+    return metric.compute(predictions=predictions, references=labels)
+
+
+# def training():
+#     dataset_id = "relbert/analogy_questions"
+#     dataset_sub = "bats"
+#     print("GETTING DATASET")
+#     raw_dataset = load_dataset(dataset_id, dataset_sub)
+#     # data_metric = evaluate.load(dataset_id, dataset_sub)
+#     checkpoint = "bert-base-uncased"
+#     model = BertModel.from_pretrained(checkpoint)
+#     # dataset = dataset["train"]
+#     # tokenized_datasets = dataset.map(tokenize_function, batched=True)
+#     # print(raw_dataset)
+#     test_data = raw_dataset["test"]
+#     # print(test_data["stem"])
+#     all_answers = []
+#     for answer in raw_dataset["answer"]:
+#         answer = raw_dataset["choice"][answer]
+#     raw_dataset = raw_dataset.add_column("label", all_answers)
+    
+        
+#     print(raw_dataset)
+#     print(raw_dataset["label"])
+#     dataset = raw_dataset.map(
+#         lambda x: tokenizer(x["stem"], truncation=True),
+#         batched=True,
+#     )
+#     print(dataset)
+#     dataset = dataset.remove_columns(["stem", "answer", "choice"])
+#     dataset = dataset.rename_column("label", "labels")
+#     dataset = dataset.with_format("torch")
+
+#     training_args = TrainingArguments("test-trainer", evaluation_strategy="epoch")
+
+#     print(dataset)
+#     # print(f"- The {dataset_id} dataset has {dataset.num_rows} examples.")
+#     # print(f"- Each example is a {type(dataset[0])} with a {type(dataset[0]['stem'])} as value.")
+#     # print(f"- Examples look like this: {dataset[0]}")
+    
+#     # small_train_dataset = tokenized_datasets["train"].shuffle(seed=42).select(range(1000))
+#     # small_eval_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(1000))
+    
+#     # dataset = dataset["train"].map(tokenize_function, batched=True)
+#     # dataset.set_format(type="torch", columns=["input_ids", "token_type_ids", "attention_mask", "label"])
+#     # dataset.format['type']
+    
+#     # tokenized_news = dataset.map(tokenize_function, batched=True)
+    
+#     # model = AutoModelForSequenceClassification.from_pretrained("sentence-transformers/all-MiniLM-L6-v2", num_labels=2)
+    
+#     # print(dataset)
+    
+#     # Choose the appropriate device based on availability (CUDA or CPU)
+#     # gpu_available = torch.cuda.is_available()
+#     # device = torch.device("cuda" if gpu_available else "cpu")
+#     # model = AutoModelForSequenceClassification.from_pretrained('bert-base-uncased')
+    
+#     # tokenized_datasets = dataset.map(tokenize_function, batched=True)
+#     # print(tokenized_datasets)
+#     # # small_train_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(1000))
+#     # # small_eval_dataset = tokenized_datasets["validation"].shuffle(seed=42).select(range(1000))
+    
+#     # model = model.to(device)
+    
+#     # model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5)
+#     # training_args = TrainingArguments(output_dir="test_trainer")
+    
+#     trainer = Trainer(
+#         model=model,
+#         args=training_args,
+#         train_dataset=dataset["test"],
+#         eval_dataset=dataset["validation"],
+#         compute_metrics=compute_metrics,
+#     )
+    
+#     output = trainer.train()
+    
+#     # train_examples = []
+#     # train_data = dataset["train"]
+#     # # For agility we only 1/2 of our available data
+#     # n_examples = dataset["train"].num_rows // 2
+    
+#     # for i in range(n_examples):
+#     #     example = train_data[i]
+#     #     # example_opposite = dataset_clean[-(i)]
+#     #     # print(example["text"])
+#     #     train_examples.append(InputExample(texts=[example['stem'], example]))
+    
+        
+#     # train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=25)
+    
+#     # print("END DATALOADER")
+    
+#     # # print(train_examples)
+        
+#     # embeddings = finetune(train_dataloader)
+#     print(output)
+    
+#     model.save("bert-analogies")
+    
+#     model.save_to_hub("smhavens/bert-base-analogies")
+#     return output
+
+
+# def finetune(train_dataloader):
+#     # model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5)
+#     model_id = "sentence-transformers/all-MiniLM-L6-v2"
+#     model = SentenceTransformer(model_id)
+#     device = torch.device('cuda:0')
+#     model = model.to(device)
+    
+#     # training_args = TrainingArguments(output_dir="test_trainer")
+    
+#     # USE THIS LINK
+#     # https://huggingface.co/blog/how-to-train-sentence-transformers
+    
+#     train_loss = losses.BatchHardSoftMarginTripletLoss(model=model)
+    
+#     print("BEGIN FIT")
+    
+#     model.fit(train_objectives=[(train_dataloader, train_loss)], epochs=10)
+    
+#     model.save("bert-analogies")
+    
+#     model.save_to_hub("smhavens/bert-base-analogies")
+#     return 0
+
+def training():
+    dataset_id = "relbert/analogy_questions"
+    dataset_sub = "bats"
+    print("GETTING DATASET")
+    dataset = load_dataset(dataset_id, dataset_sub)
+    # dataset = dataset["train"]
+    # tokenized_datasets = dataset.map(tokenize_function, batched=True)
+    
+    print(f"- The {dataset_id} dataset has {dataset['test'].num_rows} examples.")
+    print(f"- Each example is a {type(dataset['test'][0])} with a {type(dataset['test'][0]['stem'])} as value.")
+    print(f"- Examples look like this: {dataset['test'][0]}")
+    
+    train_examples = []
+    train_data = dataset["test"]
+    # For agility we only 1/2 of our available data
+    n_examples = dataset["test"].num_rows // 2
+    
+    for i in range(n_examples):
+        example = train_data[i]
+        temp_word_1 = example["stem"][0]
+        temp_word_2 = example["stem"][1]
+        temp_word_3 = example["choice"][example["answer"]][0]
+        temp_word_4 = example["choice"][example["answer"]][1]
+        comp1 = f"{temp_word_1} to {temp_word_2}"
+        comp2 = f"{temp_word_3} to {temp_word_4}"
+        # example_opposite = dataset_clean[-(i)]
+        # print(example["text"])
+        train_examples.append(InputExample(texts=[comp1, comp2]))
+    
+        
+    train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=25)
+    
+    print("END DATALOADER")
+    
+    # print(train_examples)
+        
+    embeddings = finetune(train_dataloader)
+    
+    return (dataset['test'].num_rows, type(dataset['test'][0]), type(dataset['test'][0]['stem']), dataset['test'][0], embeddings)
+
+
+def finetune(train_dataloader):
+    # model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5)
+    model_id = "sentence-transformers/all-MiniLM-L6-v2"
+    model = SentenceTransformer(model_id)
+    device = torch.device('cuda:0')
+    model = model.to(device)
+    
+    # training_args = TrainingArguments(output_dir="test_trainer")
+    
+    # USE THIS LINK
+    # https://huggingface.co/blog/how-to-train-sentence-transformers
+    
+    train_loss = losses.MegaBatchMarginLoss(model=model)
+    
+    print("BEGIN FIT")
+    
+    model.fit(train_objectives=[(train_dataloader, train_loss)], epochs=10)
+    
+    model.save("bert-analogies")
+    
+    # model.save_to_hub("smhavens/bert-base-analogies")
+    # accuracy = compute_metrics(eval, metric)
+    return 0
+
+def greet(name):
+    return "Hello " + name + "!!"
+
+def check_answer(guess:str):
+    global guesses
+    global answer
+    guesses.append(guess)
+    output = ""
+    for guess in guesses:
+        output += ("- " + guess + "\n")
+    output = output[:-1]
+    
+    if guess.lower() == answer.lower():
+        return "Correct!", output
+    else:
+        return "Try again!", output
+
+def main():
+    print("BEGIN")
+    word1 = "Black"
+    word2 = "White"
+    word3 = "Sun"
+    global answer
+    answer = "Moon"
+    global guesses
+    
+    num_rows, data_type, value, example, embeddings = training()
+    
+    # prompt = f"{word1} is to {word2} as {word3} is to ____"
+    # with gr.Blocks() as iface:
+    #     gr.Markdown(prompt)
+    #     with gr.Tab("Guess"):
+    #         text_input = gr.Textbox()
+    #         text_output = gr.Textbox()
+    #         text_button = gr.Button("Submit")
+    #     with gr.Accordion("Open for previous guesses"):
+    #         text_guesses = gr.Textbox()
+    #     with gr.Tab("Testing"):
+    #         gr.Markdown(f"""Number of rows in dataset is {num_rows}, with each having type {data_type} and value {value}.
+    #                     An example is {example}.
+    #                     The Embeddings are {embeddings}.""")
+    #     text_button.click(check_answer, inputs=[text_input], outputs=[text_output, text_guesses])
+    # # iface = gr.Interface(fn=greet, inputs="text", outputs="text")
+    # iface.launch()
+    
+    
+
+
+    
+if __name__ == "__main__":
     main()

app_context.py

@@ -1,3 +1,4 @@
+<<<<<<< HEAD
 import gradio as gr
 import math
 import spacy
@@ -251,4 +252,263 @@ def main():
 
     
 if __name__ == "__main__":
+=======
+import gradio as gr
+import math
+import spacy
+from datasets import load_dataset
+from sentence_transformers import SentenceTransformer
+from sentence_transformers import InputExample
+from sentence_transformers import losses
+from sentence_transformers import util
+from transformers import pipeline, T5Tokenizer
+from transformers import AutoTokenizer, AutoModel, AutoModelForSequenceClassification
+from transformers import TrainingArguments, Trainer, T5ForConditionalGeneration
+import torch
+import torch.nn.functional as F
+from torch.utils.data import DataLoader
+import numpy as np
+import evaluate
+import nltk
+from nltk.corpus import stopwords
+import subprocess
+import sys
+import random
+from textwrap import fill
+
+# !pip install https://huggingface.co/spacy/en_core_web_sm/resolve/main/en_core_web_sm-any-py3-none-any.whl
+subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'https://huggingface.co/spacy/en_core_web_sm/resolve/main/en_core_web_sm-any-py3-none-any.whl'])
+# tokenizer = AutoTokenizer.from_pretrained('sentence-transformers/all-MiniLM-L6-v2')
+model_base = "results/checkpoint-17000"
+nltk.download('stopwords')
+nlp = spacy.load("en_core_web_sm")
+stops = stopwords.words("english")
+ROMAN_CONSTANTS = (
+            ( "", "I", "II", "III", "IV", "V", "VI", "VII", "VIII", "IX" ),
+            ( "", "X", "XX", "XXX", "XL", "L", "LX", "LXX", "LXXX", "XC" ),
+            ( "", "C", "CC", "CCC", "CD", "D", "DC", "DCC", "DCCC", "CM" ),
+            ( "", "M", "MM", "MMM", "",   "",  "-",  "",    "",     ""   ),
+            ( "", "i", "ii", "iii", "iv", "v", "vi", "vii", "viii", "ix" ),
+            ( "", "x", "xx", "xxx", "xl", "l", "lx", "lxx", "lxxx", "xc" ),
+            ( "", "c", "cc", "ccc", "cd", "d", "dc", "dcc", "dccc", "cm" ),
+            ( "", "m", "mm", "mmm", "",   "",  "-",  "",    "",     ""   ),
+        )
+
+# answer = "Pizza"
+guesses = []
+return_guesses = []
+answer = "Moon"
+word1 = "Black"
+word2 = "White"
+word3 = "Sun"
+base_prompts = ["Sun is to Moon as ", "Black is to White as ", "Atom is to Element as",
+                "Athens is to Greece as ", "Cat is to Dog as ", "Robin is to Bird as",
+                "Hunger is to Ambition as "]
+
+
+#Mean Pooling - Take attention mask into account for correct averaging
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output['token_embeddings'] #First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+
+def normalize(comment, lowercase, remove_stopwords):
+    if lowercase:
+        comment = comment.lower()
+    comment = nlp(comment)
+    lemmatized = list()
+    for word in comment:
+        lemma = word.lemma_.strip()
+        if lemma:
+            if not remove_stopwords or (remove_stopwords and lemma not in stops):
+                lemmatized.append(lemma)
+    return " ".join(lemmatized)
+
+
+# def tokenize_function(examples):
+#     return tokenizer(examples["text"])
+
+
+def compute_metrics(eval_pred):
+    logits, labels = eval_pred
+    predictions = np.argmax(logits, axis=-1)
+    metric = evaluate.load("accuracy")
+    return metric.compute(predictions=predictions, references=labels)
+    
+    
+def get_model():
+    global model_base
+    # last_checkpoint = "./results/checkpoint-22500"
+
+    finetuned_model = T5ForConditionalGeneration.from_pretrained(model_base)
+    tokenizer = T5Tokenizer.from_pretrained(model_base)
+    # model = SentenceTransformer(model_base)
+    gpu_available = torch.cuda.is_available()
+    device = torch.device("cuda" if gpu_available else "cpu")
+    finetuned_model = finetuned_model.to(device)
+    return finetuned_model, tokenizer
+
+
+def cosine_scores(model, sentence):
+    global word1
+    global word2
+    global word3
+    # sentence1 = f"{word1} is to {word2} as"
+    embeddings1 = model.encode(sentence, convert_to_tensor=True)
+
+def embeddings(model, sentences, tokenizer):
+    global word1
+    global word2
+    global word3
+    global model_base
+    gpu_available = torch.cuda.is_available()
+    device = torch.device("cuda" if gpu_available else "cpu")
+    # device = torch.device('cuda:0')
+    # embeddings = model.encode(sentences)
+    question = "Please answer to this question: " + sentences
+    
+    inputs = tokenizer(question, return_tensors="pt")
+    
+    print(inputs)
+    # print(inputs.device)
+    print(model.device)
+    print(inputs['input_ids'].device)
+    print(inputs['attention_mask'].device)
+    
+    inputs['attention_mask'] = inputs['attention_mask'].to(device)
+    inputs['input_ids'] = inputs['input_ids'].to(device)
+    
+    outputs = model.generate(**inputs)
+    answer = tokenizer.decode(outputs[0])
+    answer = answer[6:-4]
+    # print(fill(answer, width=80))
+    
+    print("ANSWER IS", answer)
+
+    return answer
+
+
+def random_word(model, tokenizer):
+    global model_base
+    vocab = tokenizer.get_vocab()
+    # with open(model_base + '/vocab.txt', 'r') as file:
+    line = ""
+    # content = file.readlines()
+    length = tokenizer.vocab_size
+    # print(vocab)
+    while line == "":
+        rand_line = random.randrange(0, length)
+        # print("TRYING TO FIND", rand_line, "OUT OF", length, "WITH VOCAB OF TYPE", type(vocab))
+        for word, id in vocab.items():
+            if id == rand_line and word[0].isalpha() and word not in stops and word not in ROMAN_CONSTANTS:
+        # if vocab[rand_line][0].isalpha() and vocab[rand_line][:-1] not in stops and vocab[rand_line][:-1] not in ROMAN_CONSTANTS:
+                line = word
+            elif id == rand_line:
+                print(f"{word} is not alpha or is a stop word")
+    # for num, aline in enumerate(file, 1997):
+    #     if random.randrange(num) and aline.isalpha():
+    #         continue
+    #     # elif not aline.isalpha():
+            
+    #     line = aline
+    print(line)
+    return line
+
+
+def generate_prompt(model, tokenizer):
+    global word1
+    global word2
+    global word3
+    global answer
+    global base_prompts
+    word1 = random_word(model, tokenizer)
+    # word2 = random_word()
+    
+    word2 = embeddings(model, f"{base_prompts[random.randint(0, len(base_prompts) - 1)]}{word1} is to ___.", tokenizer)
+    word3 = random_word(model, tokenizer)
+    sentence = f"{word1} is to {word2} as {word3} is to ___."
+    print(sentence)
+    answer = embeddings(model, sentence, tokenizer)
+    print("ANSWER IS", answer)
+    return f"# {word1} is to {word2} as {word3} is to ___."
+    # cosine_scores(model, sentence)
+
+
+def greet(name):
+    return "Hello " + name + "!!"
+
+def check_answer(guess:str):
+    global guesses
+    global answer
+    global return_guesses
+    global word1
+    global word2
+    global word3
+    
+    model, tokenizer = get_model()
+    output = ""
+    protected_guess = guess
+    sentence = f"{word1} is to {word2} as [MASK] is to {guess}."
+   
+    other_word = embeddings(model, sentence, tokenizer)
+    guesses.append(guess)
+    
+    
+    
+    for guess in return_guesses:
+        output += ("- " + guess + "<br>")
+    
+    # output = output[:-1]
+    prompt = f"{word1} is to {word2} as {word3} is to ___."
+    # print("IS", protected_guess, "EQUAL TO", answer, ":", protected_guess.lower() == answer.lower())
+    
+    if protected_guess.lower() == answer.lower():
+        return_guesses.append(f"{protected_guess}: {word1} is to {word2} as {word3} is to {protected_guess}.")
+        output += f"<span style='color:green'>- {return_guesses[-1]}</span><br>"
+        new_prompt = generate_prompt(model, tokenizer)
+        return new_prompt, "Correct!", output
+    else:
+        return_guess = f"{protected_guess}: {word1} is to {word2} as {other_word} is to {protected_guess}."
+        return_guesses.append(return_guess)
+        output += ("- " + return_guess + " <br>")
+        return prompt, "Try again!", output
+
+def main():
+    global word1
+    global word2
+    global word3
+    global answer
+    # answer = "Moon"
+    global guesses
+    
+    
+    # num_rows, data_type, value, example, embeddings = training()
+    # sent_embeddings = embeddings()
+    model, tokenizer = get_model() 
+    generate_prompt(model, tokenizer)
+    
+    prompt = f"{word1} is to {word2} as {word3} is to ____"
+    print(prompt)
+    print("TESTING EMBEDDINGS")
+    with gr.Blocks() as iface:
+        mark_question = gr.Markdown(prompt)
+        with gr.Tab("Guess"):
+            text_input = gr.Textbox()
+            text_output = gr.Textbox()
+            text_button = gr.Button("Submit")
+        with gr.Accordion("Open for previous guesses"):
+            text_guesses = gr.Markdown()
+        # with gr.Tab("Testing"):
+        #     gr.Markdown(f"""The Embeddings are {sent_embeddings}.""")
+        text_button.click(check_answer, inputs=[text_input], outputs=[mark_question, text_output, text_guesses])
+    # iface = gr.Interface(fn=greet, inputs="text", outputs="text")
+    iface.launch()
+    
+    
+
+
+    
+if __name__ == "__main__":
+>>>>>>> 5058aea (Problems)
     main()

flan-t5-train.py

@@ -1,235 +1,235 @@
-import gradio as gr
-import math
-from datasets import load_dataset
-from transformers import AutoTokenizer, AutoModel, AutoModelForSequenceClassification
-from transformers import TrainingArguments, Trainer
-from transformers import T5Tokenizer, T5ForConditionalGeneration
-import torch
-import torch.nn.functional as F
-from torch.utils.data import DataLoader
-import numpy as np
-import evaluate
-import nltk
-from nltk.corpus import stopwords
-import subprocess
-import sys
-from transformers import T5Tokenizer, DataCollatorForSeq2Seq
-from transformers import T5ForConditionalGeneration, Seq2SeqTrainingArguments, Seq2SeqTrainer
-from transformers import DataCollatorWithPadding, DistilBertTokenizerFast
-from transformers import TrainingArguments
-from transformers import (
-    BertModel,
-    BertTokenizerFast,
-    Trainer,
-    EvalPrediction
-)
-
-nltk.download("punkt", quiet=True)
-metric = evaluate.load("rouge")
-
-# Global Parameters
-L_RATE = 3e-4
-BATCH_SIZE = 8
-PER_DEVICE_EVAL_BATCH = 4
-WEIGHT_DECAY = 0.01
-SAVE_TOTAL_LIM = 3
-NUM_EPOCHS = 10
-
-# Set up training arguments
-training_args = Seq2SeqTrainingArguments(
-   output_dir="./results",
-   evaluation_strategy="epoch",
-   learning_rate=L_RATE,
-   per_device_train_batch_size=BATCH_SIZE,
-   per_device_eval_batch_size=PER_DEVICE_EVAL_BATCH,
-   weight_decay=WEIGHT_DECAY,
-   save_total_limit=SAVE_TOTAL_LIM,
-   num_train_epochs=NUM_EPOCHS,
-   predict_with_generate=True,
-   push_to_hub=False
-)
-
-model_id = "google/flan-t5-base"
-tokenizer = T5Tokenizer.from_pretrained(model_id)
-# tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')
-# metric = evaluate.load("accuracy")
-
-def tokenize_function(examples):
-    return tokenizer(examples["stem"], padding="max_length", truncation=True)
-
-
-#Mean Pooling - Take attention mask into account for correct averaging
-def mean_pooling(model_output, attention_mask):
-    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
-    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
-    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
-
-
-# def compute_metrics(eval_pred):
-#     logits, labels = eval_pred
-#     predictions = np.argmax(logits, axis=-1)
-#     metric = evaluate.load("accuracy")
-#     return metric.compute(predictions=predictions, references=labels)
-
-def compute_metrics(eval_preds):
-   preds, labels = eval_preds
-
-   # decode preds and labels
-   labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
-   decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)
-   decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
-
-   # rougeLSum expects newline after each sentence
-   decoded_preds = ["\n".join(nltk.sent_tokenize(pred.strip())) for pred in decoded_preds]
-   decoded_labels = ["\n".join(nltk.sent_tokenize(label.strip())) for label in decoded_labels]
-
-   result = metric.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True)
-  
-   return result
-
-
-def training():
-    dataset_id = "tomasmcz/word2vec_analogy"
-    # dataset_id = "relbert/scientific_and_creative_analogy"
-    # dataset_sub = "Quadruples_Kmiecik_random_split"
-    print("GETTING DATASET")
-    dataset = load_dataset(dataset_id)
-    # dataset = dataset["train"]
-    # tokenized_datasets = dataset.map(tokenize_function, batched=True)
-    
-    print(dataset)
-    print(f"- The {dataset_id} dataset has {dataset['train'].num_rows} examples.")
-    print(f"- Each example is a {type(dataset['train'][0])} with a {type(dataset['train'][0])} as value.")
-    print(f"- Examples look like this: {dataset['train'][0]}")
-    
-    # for i in dataset["train"]:
-    #     print(i["AB"], "to", i["CD"], "is", i["label"])
-    
-    dataset = dataset["train"].train_test_split(test_size=0.3)
-    
-    # We prefix our tasks with "answer the question"
-    prefix = "Please answer this question: "
-
-        
-    def preprocess_function(examples):
-        """Add prefix to the sentences, tokenize the text, and set the labels"""
-        # The "inputs" are the tokenized answer:
-        inputs = []
-        # print(examples)
-        # inputs = [prefix + doc for doc in examples["question"]]
-        for doc in examples['word_a']:
-            # print("THE DOC IS:", doc)
-            # print("THE DOC IS:", examples[i]['AB'], examples[i]['CD'], examples[i]['label'])
-            prompt = f"{prefix}{doc} is to "
-            inputs.append(prompt)
-        # inputs = [prefix + doc for doc in examples["question"]]
-        for indx, doc in enumerate(examples["word_b"]):
-            prompt = f"{doc} as "
-            inputs[indx] += prompt
-            
-        for indx, doc in enumerate(examples["word_c"]):
-            prompt = f"{doc} is to ___."
-            inputs[indx] += prompt
-        model_inputs = tokenizer(inputs, max_length=128, truncation=True)
-        
-        # print(examples["label"], type(examples["label"]))
-        
-        # The "labels" are the tokenized outputs:
-        labels = tokenizer(text_target=examples["word_d"], 
-                            max_length=512,         
-                            truncation=True)
-
-        model_inputs["labels"] = labels["input_ids"]
-        return model_inputs
-    
-    
-    
-    # Map the preprocessing function across our dataset
-    tokenized_dataset = dataset.map(preprocess_function, batched=True)
-    
-    print("END DATALOADER")
-    
-    # print(train_examples)
-        
-    embeddings = finetune(tokenized_dataset)
-    
-    return 0
-
-
-def finetune(dataset):
-    # model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5)
-    # model_id = "sentence-transformers/all-MiniLM-L6-v2"
-    model_id = "google/flan-t5-base"
-    # model_id = "distilbert-base-uncased"
-    # tokenizer = DistilBertTokenizerFast.from_pretrained(model_id)
-    tokenizer = T5Tokenizer.from_pretrained(model_id)
-    model = T5ForConditionalGeneration.from_pretrained(model_id)
-    data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)
-    device = torch.device('cuda:0')
-    model = model.to(device)
-    
-    # training_args = TrainingArguments(output_dir="test_trainer")
-    
-    # USE THIS LINK
-    # https://huggingface.co/blog/how-to-train-sentence-transformers
-    
-    # train_loss = losses.MegaBatchMarginLoss(model=model)
-    # ds_train, ds_valid = dataset.train_test_split(test_size=0.2, seed=42)
-    
-    print("BEGIN FIT")
-    
-    trainer = Seq2SeqTrainer(
-        model=model,
-        args=training_args,
-        train_dataset=dataset["train"],
-        eval_dataset=dataset["test"],
-        # evaluation_strategy="no"
-        tokenizer=tokenizer,
-        data_collator=data_collator,
-        compute_metrics=compute_metrics
-        )
-    
-    # model.fit(train_objectives=[(train_dataloader, train_loss)], epochs=10)
-    
-    trainer.train()
-    
-    # model.save("flan-analogies")
-    
-    # model.save_to_hub("smhavens/bert-base-analogies")
-    # accuracy = compute_metrics(eval, metric)
-    return 0
-
-def greet(name):
-    return "Hello " + name + "!!"
-
-def check_answer(guess:str):
-    global guesses
-    global answer
-    guesses.append(guess)
-    output = ""
-    for guess in guesses:
-        output += ("- " + guess + "\n")
-    output = output[:-1]
-    
-    if guess.lower() == answer.lower():
-        return "Correct!", output
-    else:
-        return "Try again!", output
-
-def main():
-    print("BEGIN")
-    word1 = "Black"
-    word2 = "White"
-    word3 = "Sun"
-    global answer
-    answer = "Moon"
-    global guesses
-    
-    training()
-    
-    
-
-
-    
-if __name__ == "__main__":
+import gradio as gr
+import math
+from datasets import load_dataset
+from transformers import AutoTokenizer, AutoModel, AutoModelForSequenceClassification
+from transformers import TrainingArguments, Trainer
+from transformers import T5Tokenizer, T5ForConditionalGeneration
+import torch
+import torch.nn.functional as F
+from torch.utils.data import DataLoader
+import numpy as np
+import evaluate
+import nltk
+from nltk.corpus import stopwords
+import subprocess
+import sys
+from transformers import T5Tokenizer, DataCollatorForSeq2Seq
+from transformers import T5ForConditionalGeneration, Seq2SeqTrainingArguments, Seq2SeqTrainer
+from transformers import DataCollatorWithPadding, DistilBertTokenizerFast
+from transformers import TrainingArguments
+from transformers import (
+    BertModel,
+    BertTokenizerFast,
+    Trainer,
+    EvalPrediction
+)
+
+nltk.download("punkt", quiet=True)
+metric = evaluate.load("rouge")
+
+# Global Parameters
+L_RATE = 3e-4
+BATCH_SIZE = 8
+PER_DEVICE_EVAL_BATCH = 4
+WEIGHT_DECAY = 0.01
+SAVE_TOTAL_LIM = 3
+NUM_EPOCHS = 10
+
+# Set up training arguments
+training_args = Seq2SeqTrainingArguments(
+   output_dir="./results",
+   evaluation_strategy="epoch",
+   learning_rate=L_RATE,
+   per_device_train_batch_size=BATCH_SIZE,
+   per_device_eval_batch_size=PER_DEVICE_EVAL_BATCH,
+   weight_decay=WEIGHT_DECAY,
+   save_total_limit=SAVE_TOTAL_LIM,
+   num_train_epochs=NUM_EPOCHS,
+   predict_with_generate=True,
+   push_to_hub=False
+)
+
+model_id = "google/flan-t5-base"
+tokenizer = T5Tokenizer.from_pretrained(model_id)
+# tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')
+# metric = evaluate.load("accuracy")
+
+def tokenize_function(examples):
+    return tokenizer(examples["stem"], padding="max_length", truncation=True)
+
+
+#Mean Pooling - Take attention mask into account for correct averaging
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+
+# def compute_metrics(eval_pred):
+#     logits, labels = eval_pred
+#     predictions = np.argmax(logits, axis=-1)
+#     metric = evaluate.load("accuracy")
+#     return metric.compute(predictions=predictions, references=labels)
+
+def compute_metrics(eval_preds):
+   preds, labels = eval_preds
+
+   # decode preds and labels
+   labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
+   decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)
+   decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
+
+   # rougeLSum expects newline after each sentence
+   decoded_preds = ["\n".join(nltk.sent_tokenize(pred.strip())) for pred in decoded_preds]
+   decoded_labels = ["\n".join(nltk.sent_tokenize(label.strip())) for label in decoded_labels]
+
+   result = metric.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True)
+  
+   return result
+
+
+def training():
+    dataset_id = "tomasmcz/word2vec_analogy"
+    # dataset_id = "relbert/scientific_and_creative_analogy"
+    # dataset_sub = "Quadruples_Kmiecik_random_split"
+    print("GETTING DATASET")
+    dataset = load_dataset(dataset_id)
+    # dataset = dataset["train"]
+    # tokenized_datasets = dataset.map(tokenize_function, batched=True)
+    
+    print(dataset)
+    print(f"- The {dataset_id} dataset has {dataset['train'].num_rows} examples.")
+    print(f"- Each example is a {type(dataset['train'][0])} with a {type(dataset['train'][0])} as value.")
+    print(f"- Examples look like this: {dataset['train'][0]}")
+    
+    # for i in dataset["train"]:
+    #     print(i["AB"], "to", i["CD"], "is", i["label"])
+    
+    dataset = dataset["train"].train_test_split(test_size=0.3)
+    
+    # We prefix our tasks with "answer the question"
+    prefix = "Please answer this question: "
+
+        
+    def preprocess_function(examples):
+        """Add prefix to the sentences, tokenize the text, and set the labels"""
+        # The "inputs" are the tokenized answer:
+        inputs = []
+        # print(examples)
+        # inputs = [prefix + doc for doc in examples["question"]]
+        for doc in examples['word_a']:
+            # print("THE DOC IS:", doc)
+            # print("THE DOC IS:", examples[i]['AB'], examples[i]['CD'], examples[i]['label'])
+            prompt = f"{prefix}{doc} is to "
+            inputs.append(prompt)
+        # inputs = [prefix + doc for doc in examples["question"]]
+        for indx, doc in enumerate(examples["word_b"]):
+            prompt = f"{doc} as "
+            inputs[indx] += prompt
+            
+        for indx, doc in enumerate(examples["word_c"]):
+            prompt = f"{doc} is to ___."
+            inputs[indx] += prompt
+        model_inputs = tokenizer(inputs, max_length=128, truncation=True)
+        
+        # print(examples["label"], type(examples["label"]))
+        
+        # The "labels" are the tokenized outputs:
+        labels = tokenizer(text_target=examples["word_d"], 
+                            max_length=512,         
+                            truncation=True)
+
+        model_inputs["labels"] = labels["input_ids"]
+        return model_inputs
+    
+    
+    
+    # Map the preprocessing function across our dataset
+    tokenized_dataset = dataset.map(preprocess_function, batched=True)
+    
+    print("END DATALOADER")
+    
+    # print(train_examples)
+        
+    embeddings = finetune(tokenized_dataset)
+    
+    return 0
+
+
+def finetune(dataset):
+    # model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5)
+    # model_id = "sentence-transformers/all-MiniLM-L6-v2"
+    model_id = "google/flan-t5-base"
+    # model_id = "distilbert-base-uncased"
+    # tokenizer = DistilBertTokenizerFast.from_pretrained(model_id)
+    tokenizer = T5Tokenizer.from_pretrained(model_id)
+    model = T5ForConditionalGeneration.from_pretrained(model_id)
+    data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)
+    device = torch.device('cuda:0')
+    model = model.to(device)
+    
+    # training_args = TrainingArguments(output_dir="test_trainer")
+    
+    # USE THIS LINK
+    # https://huggingface.co/blog/how-to-train-sentence-transformers
+    
+    # train_loss = losses.MegaBatchMarginLoss(model=model)
+    # ds_train, ds_valid = dataset.train_test_split(test_size=0.2, seed=42)
+    
+    print("BEGIN FIT")
+    
+    trainer = Seq2SeqTrainer(
+        model=model,
+        args=training_args,
+        train_dataset=dataset["train"],
+        eval_dataset=dataset["test"],
+        # evaluation_strategy="no"
+        tokenizer=tokenizer,
+        data_collator=data_collator,
+        compute_metrics=compute_metrics
+        )
+    
+    # model.fit(train_objectives=[(train_dataloader, train_loss)], epochs=10)
+    
+    trainer.train()
+    
+    # model.save("flan-analogies")
+    
+    # model.save_to_hub("smhavens/bert-base-analogies")
+    # accuracy = compute_metrics(eval, metric)
+    return 0
+
+def greet(name):
+    return "Hello " + name + "!!"
+
+def check_answer(guess:str):
+    global guesses
+    global answer
+    guesses.append(guess)
+    output = ""
+    for guess in guesses:
+        output += ("- " + guess + "\n")
+    output = output[:-1]
+    
+    if guess.lower() == answer.lower():
+        return "Correct!", output
+    else:
+        return "Try again!", output
+
+def main():
+    print("BEGIN")
+    word1 = "Black"
+    word2 = "White"
+    word3 = "Sun"
+    global answer
+    answer = "Moon"
+    global guesses
+    
+    training()
+    
+    
+
+
+    
+if __name__ == "__main__":
     main()

word_embedding.py

@@ -1,4 +1,7 @@
 <<<<<<< HEAD
+<<<<<<< HEAD
+=======
+>>>>>>> 5058aea (Problems)
 from datasets import load_dataset
 import shutil
 import json
@@ -615,6 +618,7 @@ def main():
     
 
 if __name__ == "__main__":
+<<<<<<< HEAD
 =======
 from datasets import load_dataset
 import shutil
@@ -1233,4 +1237,6 @@ def main():
 
 if __name__ == "__main__":
 >>>>>>> 7d5b505 (New in-context model with working UI System)
+=======
+>>>>>>> 5058aea (Problems)
     main()