Rename psy.ipynb to psy.py

psy.ipynb

@@ -1,4 +0,0 @@
-
-
-
-

psy.py

@@ -0,0 +1,21 @@
+from datasets import load_dataset
+from transformers import AutoTokenizer
+
+DATA_SEED = 9843203
+QUICK_TEST = True
+
+# This is our baseline dataset
+dataset = load_dataset("ClaudiaRichard/mbti_classification_v2")
+
+# LLama3 8b
+tokeniser = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B")
+
+def tokenise_function(examples):
+    return tokeniser(examples["text"], padding="max_length", truncation=True)
+
+tokenised_dataset = dataset.map(tokenise_function, batched=True)
+
+
+# Different sized datasets will allow for different training times
+train_dataset = tokenized_datasets["train"].shuffle(seed=DATA_SEED).select(range(1000)) if QUICK_TEST  else  tokenized_datasets["train"].shuffle(seed=DATA_SEED)
+test_dataset = tokenized_datasets["test"].shuffle(seed=DATA_SEED).select(range(1000)) if QUICK_TEST else tokenized_datasets["test"].shuffle(seed=DATA_SEED)