Uploaded project

config.json

@@ -0,0 +1,46 @@
+{
+  "_name_or_path": "distilgpt2",
+  "_num_labels": 1,
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "id2label": {
+    "0": "LABEL_0"
+  },
+  "initializer_range": 0.02,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 1024,
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 6,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_specific_params": {
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 50
+    }
+  },
+  "torch_dtype": "float32",
+  "transformers_version": "4.25.1",
+  "use_cache": true,
+  "vocab_size": 50257
+}

generation.py

@@ -0,0 +1,12 @@
+from transformers import AutoTokenizer
+from transformers import AutoModelWithLMHead
+from transformers import GPT2LMHeadModel
+
+FOLDER_NAME = "./distilgpt2_quotes.TRANS"
+model: GPT2LMHeadModel = AutoModelWithLMHead.from_pretrained(FOLDER_NAME)
+tokenizer = AutoTokenizer.from_pretrained("distilgpt2")
+
+text = "Plato: "
+ids = tokenizer(text, return_tensors = "pt")
+out = model.generate(**ids, early_stopping = True, max_length = 90, do_sample = True)
+print(tokenizer.decode(out[0]))

generator1.py

@@ -0,0 +1,31 @@
+from transformers import AutoTokenizer
+from transformers import AutoModelWithLMHead
+from transformers import Trainer, TrainingArguments
+from transformers import DataCollatorForLanguageModeling
+
+from datasets import load_dataset
+
+
+data = load_dataset("json", data_files = "./authors_all_CUT.json")
+data = data["train"].train_test_split(test_size = 0.10)
+
+tokenizer = AutoTokenizer.from_pretrained("distilgpt2")
+tokenizer.pad_token = tokenizer.eos_token #Quick fix to an issue; DistilGPT2 does not include a padding token
+def tokenize_datasets(data_set):
+    return tokenizer(data_set["text"], padding = False, truncation = True)
+
+BATCH_SIZE = 8
+data = data.map(tokenize_datasets, batched = True, batch_size = BATCH_SIZE) #Tokenize and batch all text
+
+
+FOLDER_NAME = "./distilgpt2_quotes.TRANS"
+model = AutoModelWithLMHead.from_pretrained("distilgpt2")
+#model = AutoModelWithLMHead.from_pretrained(FOLDER_NAME)
+collator = DataCollatorForLanguageModeling(tokenizer, mlm = False)
+
+EPOCHS = 5
+training_args = TrainingArguments(FOLDER_NAME, overwrite_output_dir = True, num_train_epochs = EPOCHS, per_device_train_batch_size = BATCH_SIZE, per_device_eval_batch_size = BATCH_SIZE, eval_steps = 400, save_steps = 800)
+trainer = Trainer(model, args = training_args, data_collator = collator, train_dataset = data["train"], eval_dataset = data["test"])
+
+trainer.train()
+trainer.save_model()

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04d4f3bafd9dcf92232437df75a31b0e0c5cfb5f5611b31185862d98040a005a
+size 333969117

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b96db598d5dcd97cb51471c770578b42a77fd3f5160b06bdebc03030e57bb8dd
+size 3375