{"cells":[{"cell_type":"code","execution_count":1,"metadata":{"executionInfo":{"elapsed":1223,"status":"ok","timestamp":1682264035599,"user":{"displayName":"JAIME VILLA PLAZA","userId":"16556643678799076434"},"user_tz":-120},"id":"GqjpNTL0lXw3"},"outputs":[{"name":"stdout","output_type":"stream","text":["757\n"]}],"source":["from dataset import GOOGLE, GOODBYE, CHAT, VISION, GITHUB\n","data = GOOGLE + GOODBYE + CHAT + VISION + GITHUB\n","\n","print(len(data))\n","    \n"]},{"cell_type":"code","execution_count":5,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Requirement already satisfied: transformers[torch] in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (4.30.2)\n","Requirement already satisfied: filelock in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from transformers[torch]) (3.12.2)\n","Requirement already satisfied: huggingface-hub<1.0,>=0.14.1 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from transformers[torch]) (0.15.1)\n","Requirement already satisfied: numpy>=1.17 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from transformers[torch]) (1.23.5)\n","Requirement already satisfied: packaging>=20.0 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from transformers[torch]) (23.1)\n","Requirement already satisfied: pyyaml>=5.1 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from transformers[torch]) (6.0)\n","Requirement already satisfied: regex!=2019.12.17 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from transformers[torch]) (2023.6.3)\n","Requirement already satisfied: requests in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from transformers[torch]) (2.31.0)\n","Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from transformers[torch]) (0.13.3)\n","Requirement already satisfied: safetensors>=0.3.1 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from transformers[torch]) (0.3.1)\n","Requirement already satisfied: tqdm>=4.27 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from transformers[torch]) (4.65.0)\n","Requirement already satisfied: torch!=1.12.0,>=1.9 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from transformers[torch]) (2.0.1)\n","Requirement already satisfied: accelerate>=0.20.2 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from transformers[torch]) (0.20.3)\n","Requirement already satisfied: psutil in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from accelerate>=0.20.2->transformers[torch]) (5.9.0)\n","Requirement already satisfied: fsspec in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from huggingface-hub<1.0,>=0.14.1->transformers[torch]) (2023.6.0)\n","Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from huggingface-hub<1.0,>=0.14.1->transformers[torch]) (4.7.1)\n","Requirement already satisfied: sympy in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from torch!=1.12.0,>=1.9->transformers[torch]) (1.12)\n","Requirement already satisfied: networkx in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from torch!=1.12.0,>=1.9->transformers[torch]) (3.1)\n","Requirement already satisfied: jinja2 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from torch!=1.12.0,>=1.9->transformers[torch]) (3.1.2)\n","Requirement already satisfied: colorama in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from tqdm>=4.27->transformers[torch]) (0.4.6)\n","Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from requests->transformers[torch]) (3.1.0)\n","Requirement already satisfied: idna<4,>=2.5 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from requests->transformers[torch]) (3.4)\n","Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from requests->transformers[torch]) (1.26.16)\n","Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from requests->transformers[torch]) (2023.5.7)\n","Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from jinja2->torch!=1.12.0,>=1.9->transformers[torch]) (2.1.3)\n","Requirement already satisfied: mpmath>=0.19 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from sympy->torch!=1.12.0,>=1.9->transformers[torch]) (1.3.0)\n","Note: you may need to restart the kernel to use updated packages.\n","Requirement already satisfied: accelerate in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (0.20.3)\n","Requirement already satisfied: numpy>=1.17 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from accelerate) (1.23.5)\n","Requirement already satisfied: packaging>=20.0 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from accelerate) (23.1)\n","Requirement already satisfied: psutil in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from accelerate) (5.9.0)\n","Requirement already satisfied: pyyaml in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from accelerate) (6.0)\n","Requirement already satisfied: torch>=1.6.0 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from accelerate) (2.0.1)\n","Requirement already satisfied: filelock in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from torch>=1.6.0->accelerate) (3.12.2)\n","Requirement already satisfied: typing-extensions in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from torch>=1.6.0->accelerate) (4.7.1)\n","Requirement already satisfied: sympy in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from torch>=1.6.0->accelerate) (1.12)\n","Requirement already satisfied: networkx in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from torch>=1.6.0->accelerate) (3.1)\n","Requirement already satisfied: jinja2 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from torch>=1.6.0->accelerate) (3.1.2)\n","Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from jinja2->torch>=1.6.0->accelerate) (2.1.3)\n","Requirement already satisfied: mpmath>=0.19 in c:\\users\\s9053161\\documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages (from sympy->torch>=1.6.0->accelerate) (1.3.0)\n","Note: you may need to restart the kernel to use updated packages.\n"]}],"source":["%pip install transformers[torch]\n","%pip install accelerate -U"]},{"cell_type":"code","execution_count":2,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":573},"executionInfo":{"elapsed":126030,"status":"ok","timestamp":1682264960652,"user":{"displayName":"JAIME VILLA PLAZA","userId":"16556643678799076434"},"user_tz":-120},"id":"VblJOqkFoJHe","outputId":"b3c1f423-6624-4b14-dadf-68c7adec20cb"},"outputs":[{"name":"stderr","output_type":"stream","text":["c:\\Users\\S9053161\\Documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n","  from .autonotebook import tqdm as notebook_tqdm\n","Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.bias', 'cls.seq_relationship.weight']\n","- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n","- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n","Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n","c:\\Users\\S9053161\\Documents\\projects\\gpt-voice-assistant\\.conda\\lib\\site-packages\\transformers\\optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n","  warnings.warn(\n","                                                \n"," 10%|█         | 76/760 [04:08<33:53,  2.97s/it]"]},{"name":"stdout","output_type":"stream","text":["{'eval_loss': 0.030141128227114677, 'eval_runtime': 10.8983, 'eval_samples_per_second': 13.947, 'eval_steps_per_second': 1.743, 'epoch': 1.0}\n"]},{"name":"stderr","output_type":"stream","text":["                                                  \n"," 20%|██        | 152/760 [08:26<31:02,  3.06s/it]"]},{"name":"stdout","output_type":"stream","text":["{'eval_loss': 0.008563311770558357, 'eval_runtime': 12.2288, 'eval_samples_per_second': 12.43, 'eval_steps_per_second': 1.554, 'epoch': 2.0}\n"]},{"name":"stderr","output_type":"stream","text":["                                                   \n"," 30%|███       | 228/760 [12:35<27:06,  3.06s/it]"]},{"name":"stdout","output_type":"stream","text":["{'eval_loss': 0.04210153967142105, 'eval_runtime': 12.9654, 'eval_samples_per_second': 11.724, 'eval_steps_per_second': 1.465, 'epoch': 3.0}\n"]},{"name":"stderr","output_type":"stream","text":["                                                   \n"," 40%|████      | 304/760 [16:51<23:59,  3.16s/it]"]},{"name":"stdout","output_type":"stream","text":["{'eval_loss': 0.016036802902817726, 'eval_runtime': 10.8075, 'eval_samples_per_second': 14.064, 'eval_steps_per_second': 1.758, 'epoch': 4.0}\n"]},{"name":"stderr","output_type":"stream","text":["                                                 \n"," 50%|█████     | 380/760 [20:47<17:27,  2.76s/it]"]},{"name":"stdout","output_type":"stream","text":["{'eval_loss': 0.01568855531513691, 'eval_runtime': 10.5235, 'eval_samples_per_second': 14.444, 'eval_steps_per_second': 1.805, 'epoch': 5.0}\n"]},{"name":"stderr","output_type":"stream","text":["                                                 \n"," 60%|██████    | 456/760 [25:03<13:46,  2.72s/it]"]},{"name":"stdout","output_type":"stream","text":["{'eval_loss': 0.0158185176551342, 'eval_runtime': 10.4724, 'eval_samples_per_second': 14.514, 'eval_steps_per_second': 1.814, 'epoch': 6.0}\n"]},{"name":"stderr","output_type":"stream","text":[" 66%|██████▌   | 500/760 [27:32<15:42,  3.63s/it]"]},{"name":"stdout","output_type":"stream","text":["{'loss': 0.0954, 'learning_rate': 1.7105263157894737e-05, 'epoch': 6.58}\n"]},{"name":"stderr","output_type":"stream","text":["                                                 \n"," 70%|███████   | 532/760 [29:25<11:00,  2.90s/it]"]},{"name":"stdout","output_type":"stream","text":["{'eval_loss': 0.01628260686993599, 'eval_runtime': 10.8747, 'eval_samples_per_second': 13.977, 'eval_steps_per_second': 1.747, 'epoch': 7.0}\n"]},{"name":"stderr","output_type":"stream","text":["                                                 \n"," 80%|████████  | 608/760 [33:35<07:13,  2.85s/it]"]},{"name":"stdout","output_type":"stream","text":["{'eval_loss': 0.016459450125694275, 'eval_runtime': 12.6362, 'eval_samples_per_second': 12.029, 'eval_steps_per_second': 1.504, 'epoch': 8.0}\n"]},{"name":"stderr","output_type":"stream","text":["                                                 \n"," 90%|█████████ | 684/760 [38:00<03:55,  3.09s/it]"]},{"name":"stdout","output_type":"stream","text":["{'eval_loss': 0.01655273139476776, 'eval_runtime': 11.1512, 'eval_samples_per_second': 13.631, 'eval_steps_per_second': 1.704, 'epoch': 9.0}\n"]},{"name":"stderr","output_type":"stream","text":["                                                 \n","100%|██████████| 760/760 [42:00<00:00,  2.67s/it]"]},{"name":"stdout","output_type":"stream","text":["{'eval_loss': 0.016479341313242912, 'eval_runtime': 10.1493, 'eval_samples_per_second': 14.976, 'eval_steps_per_second': 1.872, 'epoch': 10.0}\n"]},{"name":"stderr","output_type":"stream","text":["100%|██████████| 760/760 [42:01<00:00,  3.32s/it]"]},{"name":"stdout","output_type":"stream","text":["{'train_runtime': 2521.8708, 'train_samples_per_second': 2.399, 'train_steps_per_second': 0.301, 'train_loss': 0.06297851167619228, 'epoch': 10.0}\n"]},{"name":"stderr","output_type":"stream","text":["\n"]},{"data":{"text/plain":["TrainOutput(global_step=760, training_loss=0.06297851167619228, metrics={'train_runtime': 2521.8708, 'train_samples_per_second': 2.399, 'train_steps_per_second': 0.301, 'train_loss': 0.06297851167619228, 'epoch': 10.0})"]},"execution_count":2,"metadata":{},"output_type":"execute_result"}],"source":["import torch\n","from torch.utils.data import Dataset\n","from transformers import AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer\n","from sklearn.model_selection import train_test_split\n","\n","class CustomDataset(Dataset):\n","    def __init__(self, encodings, labels):\n","        self.encodings = encodings\n","        self.labels = labels\n","\n","    def __getitem__(self, idx):\n","        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n","        item['labels'] = torch.tensor(self.labels[idx])\n","        return item\n","\n","    def __len__(self):\n","        return len(self.labels)\n","\n","# Load the tokenizer and the model\n","tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')\n","model = AutoModelForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=5)\n","\n","# Prepare the dataset\n","texts = [item[0] for item in data]\n","labels = [item[1] for item in data]\n","label_map = {'vision': 0, 'chat': 1, 'goodbye': 2, 'google': 3, 'github': 4}\n","labels = [label_map[label] for label in labels]\n","\n","# Split the dataset into training and validation sets\n","train_texts, val_texts, train_labels, val_labels = train_test_split(texts, labels, test_size=0.2, random_state=42)\n","\n","# Tokenize the text\n","train_encodings = tokenizer(train_texts, truncation=True, padding=True)\n","val_encodings = tokenizer(val_texts, truncation=True, padding=True)\n","\n","# Create the custom dataset\n","train_dataset = CustomDataset(train_encodings, train_labels)\n","val_dataset = CustomDataset(val_encodings, val_labels)\n","\n","# Create the Trainer\n","training_args = TrainingArguments(\n","    output_dir='../models',\n","    num_train_epochs=10,\n","    per_device_train_batch_size=8,\n","    per_device_eval_batch_size=8,\n","    logging_dir='./logs',\n","    learning_rate=5e-5,\n","    save_total_limit=1,\n","    evaluation_strategy=\"epoch\",\n","    save_strategy=\"epoch\",  # Save a checkpoint at the end of each epoch\n",")\n","\n","\n","trainer = Trainer(\n","    model=model,\n","    args=training_args,\n","    train_dataset=train_dataset,\n","    eval_dataset=val_dataset,\n",")\n","\n","# Fine-tune the model\n","trainer.train()\n"]},{"cell_type":"code","execution_count":4,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":5146,"status":"ok","timestamp":1682264988626,"user":{"displayName":"JAIME VILLA PLAZA","userId":"16556643678799076434"},"user_tz":-120},"id":"zsmKH6j0qvzJ","outputId":"ae99c575-a283-4b1f-8419-0a2c547203f2"},"outputs":[{"name":"stderr","output_type":"stream","text":["Xformers is not installed correctly. If you want to use memory_efficient_attention to accelerate training use the following command to install Xformers\n","pip install xformers.\n"]},{"name":"stdout","output_type":"stream","text":["Hello there! : chat\n","I'd like you to tell me about powerlifting : chat\n","Can you see me? : vision\n","What do you see in this image? : vision\n","See you tomorrow! : goodbye\n","Goodbye GPT : goodbye\n","What is a compiled programing language? : google\n","How many calories does Ultra White Monster Energy have? : google\n","Let's create a new project : github\n","I want to open a new repo : github\n"]}],"source":["from transformers import pipeline\n","\n","# Load the fine-tuned model\n","# model_path = '../models/cd_CKPT_V'\n","model_path = '../models\\checkpoint-760'\n","tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')\n","classifier = pipeline('text-classification', model=model_path, tokenizer=tokenizer)\n","\n","def command_filter(prompt):\n","    # Classify the input prompt\n","    result = classifier(prompt)\n","    command_id = int(result[0]['label'].split('_')[-1])\n","    command = {0: 'vision', 1: 'chat', 2: 'goodbye', 3: 'google', 4: 'github'}[command_id]\n","\n","    return command\n","    \n","# Example prompts\n","\n","prompts = [\"Hello there!\",\n","           \"I'd like you to tell me about powerlifting\",\n","           \"Can you see me?\",\n","           \"What do you see in this image?\",\n","           \"See you tomorrow!\",\n","           \"Goodbye GPT\",\n","           \"What is a compiled programing language?\",\n","           \"How many calories does Ultra White Monster Energy have?\",\n","           \"Let's create a new project\",\n","           \"I want to open a new repo\"]\n","\n","for prompt in prompts:\n","\n","\n","    print(f'{prompt} : {command_filter(prompt)}')\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]}],"metadata":{"accelerator":"GPU","colab":{"authorship_tag":"ABX9TyMA1LT8Sj9ffscvm4bHdwRG","mount_file_id":"1tIekOnaB887ksJ8Tvr5AF9EbQSmgwHs2","provenance":[]},"gpuClass":"standard","kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.11"}},"nbformat":4,"nbformat_minor":0}