Updated training script and data

app.py

@@ -2,7 +2,8 @@ from transformers import GPT2Tokenizer, GPT2LMHeadModel, Trainer, TrainingArgume
 from datasets import load_dataset
 
 # Load the text dataset from the specified file
-dataset = load_dataset("text", data_files="training.txt")
+dataset = load_dataset("text", data_files="C:\\Users\\Mike Corrigan\\Documents\\DMD Year 4 Sem 1\\FYP INFO\\fyp hugging face\\fypmc20277423\\training.txt")
+
 
 # Initialize the GPT-2 tokenizer
 tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
@@ -37,7 +38,7 @@ training_args = TrainingArguments(
     overwrite_output_dir=True,
     num_train_epochs=2,  # Optionally reduced for quicker iteration
     per_device_train_batch_size=2,  # Reduced from 4 to 2
-    gradient_accumulation_steps=8,  # Added to compensate for smaller batch size
+    gradient_accumulation_steps=16,  # Added to compensate for smaller batch size
     save_steps=10_000,
     save_total_limit=2,
 )
@@ -52,6 +53,6 @@ trainer = Trainer(
 # Start the training process
 trainer.train()
 
-# Save the fine-tuned model and tokenizer
-model.save_pretrained("fine_tuned_gpt2_model")
-tokenizer.save_pretrained("fine_tuned_gpt2_model")
+model.save_pretrained('C:\\Users\\Mike Corrigan\\Documents\\DMD Year 4 Sem 1\\FYP INFO\\fyp hugging face\\fypmc20277423\\output\\fine_tuned_gpt2_model')
+tokenizer.save_pretrained('C:\\Users\\Mike Corrigan\\Documents\\DMD Year 4 Sem 1\\FYP INFO\\fyp hugging face\\fypmc20277423\\output\\fine_tuned_gpt2_model')
+

fine_tuned_gpt2_model/config.json

@@ -1,39 +0,0 @@
-{
-  "_name_or_path": "gpt2",
-  "activation_function": "gelu_new",
-  "architectures": [
-    "GPT2LMHeadModel"
-  ],
-  "attn_pdrop": 0.1,
-  "bos_token_id": 50256,
-  "embd_pdrop": 0.1,
-  "eos_token_id": 50256,
-  "initializer_range": 0.02,
-  "layer_norm_epsilon": 1e-05,
-  "model_type": "gpt2",
-  "n_ctx": 1024,
-  "n_embd": 768,
-  "n_head": 12,
-  "n_inner": null,
-  "n_layer": 12,
-  "n_positions": 1024,
-  "reorder_and_upcast_attn": false,
-  "resid_pdrop": 0.1,
-  "scale_attn_by_inverse_layer_idx": false,
-  "scale_attn_weights": true,
-  "summary_activation": null,
-  "summary_first_dropout": 0.1,
-  "summary_proj_to_labels": true,
-  "summary_type": "cls_index",
-  "summary_use_proj": true,
-  "task_specific_params": {
-    "text-generation": {
-      "do_sample": true,
-      "max_length": 50
-    }
-  },
-  "torch_dtype": "float32",
-  "transformers_version": "4.39.2",
-  "use_cache": true,
-  "vocab_size": 50257
-}

fine_tuned_gpt2_model/generation_config.json

@@ -1,6 +0,0 @@
-{
-  "_from_model_config": true,
-  "bos_token_id": 50256,
-  "eos_token_id": 50256,
-  "transformers_version": "4.39.2"
-}

fine_tuned_gpt2_model/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ff5c53b27a7bc018d4cd08d406c1d9cfa150b4a3fac682703ff231d8408bc205
-size 497774208

fine_tuned_gpt2_model/special_tokens_map.json

@@ -1,24 +0,0 @@
-{
-  "bos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": "<|endoftext|>",
-  "unk_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  }
-}

fine_tuned_gpt2_model/tokenizer_config.json

@@ -1,22 +0,0 @@
-{
-  "add_bos_token": false,
-  "add_prefix_space": false,
-  "added_tokens_decoder": {
-    "50256": {
-      "content": "<|endoftext|>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "bos_token": "<|endoftext|>",
-  "clean_up_tokenization_spaces": true,
-  "eos_token": "<|endoftext|>",
-  "errors": "replace",
-  "model_max_length": 1024,
-  "pad_token": "<|endoftext|>",
-  "tokenizer_class": "GPT2Tokenizer",
-  "unk_token": "<|endoftext|>"
-}

fypmc20277423/.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

fypmc20277423/README.md

@@ -1,12 +0,0 @@
----
-title: Fypmc20277423
-emoji: 📚
-colorFrom: indigo
-colorTo: gray
-sdk: streamlit
-sdk_version: 1.32.0
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

fypmc20277423/app.py

@@ -1,52 +0,0 @@
-from transformers import GPT2Tokenizer, GPT2LMHeadModel, Trainer, TrainingArguments
-from datasets import load_dataset
-
-# Load the text dataset from the specified file.
-dataset = load_dataset("text", data_files="training.txt")
-
-# Initialize the GPT-2 tokenizer.
-tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
-# Set the tokenizer's pad token to the EOS token.
-tokenizer.pad_token = tokenizer.eos_token
-
-# Define a function to tokenize the dataset and prepare labels.
-def tokenize_function(examples):
-    # Tokenize the text to input_ids, attention_mask
-    tokenized_inputs = tokenizer(examples["text"], padding="max_length", truncation=True, max_length=512)
-    # Prepare labels: labels are the same as input_ids for language modeling
-    tokenized_inputs["labels"] = tokenized_inputs["input_ids"].copy()
-    return tokenized_inputs
-
-# Tokenize the entire dataset.
-tokenized_datasets = dataset.map(tokenize_function, batched=True)
-# Remove the 'text' column as it's no longer needed after tokenization.
-tokenized_datasets = tokenized_datasets.remove_columns(["text"])
-# Set the format of the dataset to PyTorch tensors.
-tokenized_datasets.set_format(type="torch", columns=["input_ids", "attention_mask", "labels"])
-
-# Load the GPT-2 model.
-model = GPT2LMHeadModel.from_pretrained("gpt2")
-
-# Define training arguments.
-training_args = TrainingArguments(
-    output_dir="./output",
-    overwrite_output_dir=True,
-    num_train_epochs=3,
-    per_device_train_batch_size=4,
-    save_steps=10_000,
-    save_total_limit=2,
-)
-
-# Initialize the Trainer with the training dataset including labels.
-trainer = Trainer(
-    model=model,
-    args=training_args,
-    train_dataset=tokenized_datasets["train"],
-)
-
-# Start the training process.
-trainer.train()
-
-# Save the fine-tuned model and tokenizer.
-model.save_pretrained("fine_tuned_gpt2_model")
-tokenizer.save_pretrained("fine_tuned_gpt2_model")

fypmc20277423/output/runs/Mar31_21-01-46_LAPTOP-23I4320M/events.out.tfevents.1711915307.LAPTOP-23I4320M.22972.0

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7f422cc424360ad0b6d022f492ce2d8d45fa8fb317ad09588d920344d1bfdaad
-size 4805

fypmc20277423/requirements.txt

@@ -1,4 +0,0 @@
-torch
-transformers[torch]
-streamlit
-accelerate>=0.21.0