{ "cells": [ { "cell_type": "code", "execution_count": 74, "id": "98bfd1a3-ef1c-4bb7-a6ac-e27233279605", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Tue Jul 25 08:32:04 2023 \n", "+-----------------------------------------------------------------------------+\n", "| NVIDIA-SMI 525.85.12 Driver Version: 525.85.12 CUDA Version: 12.0 |\n", "|-------------------------------+----------------------+----------------------+\n", "| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n", "| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n", "| | | MIG M. |\n", "|===============================+======================+======================|\n", "| 0 NVIDIA A10 On | 00000000:06:00.0 Off | 0 |\n", "| 0% 68C P0 74W / 150W | 12938MiB / 23028MiB | 0% Default |\n", "| | | N/A |\n", "+-------------------------------+----------------------+----------------------+\n", " \n", "+-----------------------------------------------------------------------------+\n", "| Processes: |\n", "| GPU GI CI PID Type Process name GPU Memory |\n", "| ID ID Usage |\n", "|=============================================================================|\n", "| 0 N/A N/A 78960 C /usr/bin/python3 12936MiB |\n", "+-----------------------------------------------------------------------------+\n" ] } ], "source": [ "!nvidia-smi" ] }, { "cell_type": "code", "execution_count": 2, "id": "e3f5bcc9-0b62-4b6b-b12d-d70840f86ab9", "metadata": {}, "outputs": [], "source": [ "!pip install -Uqqq pip --progress-bar off\n", "!pip install -Uqqq bitsandbytes --progress-bar off\n", "!pip install -Uqqq torch==2.0.1 --progress-bar off\n", "!pip install -Uqqq git+https://github.com/huggingface/transformers.git@e03a9cc --progress-bar off\n", "!pip install -Uqqq git+https://github.com/huggingface/peft.git@42a184f --progress-bar off\n", "!pip install -Uqqq git+https://github.com/huggingface/accelerate.git@c9fbb71 --progress-bar off\n", "!pip install -Uqqq datasets==2.12.0 --progress-bar off\n", "!pip install -Uqqq loralib==0.1.1 --progress-bar off\n", "!pip install -Uqqq einops==0.6.1 --progress-bar off" ] }, { "cell_type": "code", "execution_count": 3, "id": "c97b6c51-4352-4089-9a0f-c840452fe019", "metadata": {}, "outputs": [], "source": [ "!pip install jsonschema==3.0.2 >/dev/null\n", "!pip install transforms >/dev/null" ] }, { "cell_type": "code", "execution_count": 1, "id": "88ff73a5-f8ba-4ced-ac9d-d1b95f0c1761", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/ubuntu/.local/lib/python3.8/site-packages/pandas/core/computation/expressions.py:20: UserWarning: Pandas requires version '2.7.3' or newer of 'numexpr' (version '2.7.1' currently installed).\n", " from pandas.core.computation.check import NUMEXPR_INSTALLED\n" ] } ], "source": [ "import json\n", "import os\n", "from pprint import pprint\n", "\n", "import bitsandbytes as bnb\n", "import pandas as pd\n", "import torch\n", "import torch.nn as nn\n", "import transformers\n", "from datasets import load_dataset\n", "from huggingface_hub import notebook_login\n", "from peft import (\n", " LoraConfig,\n", " PeftConfig,\n", " PeftModel,\n", " get_peft_model,\n", " prepare_model_for_kbit_training,\n", ")\n", "from transformers import (\n", " AutoConfig,\n", " AutoModelForCausalLM,\n", " AutoTokenizer,\n", " BitsAndBytesConfig,\n", ")\n", "os.environ['CUDA_VISIBLE_DEVICES'] = '0'" ] }, { "cell_type": "code", "execution_count": 2, "id": "77c8918e-ad02-45a8-8f21-1b785fea806d", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "599dcfa15df5412cab2921d5dffc7024", "version_major": 2, "version_minor": 0 }, "text/plain": [ "VBox(children=(HTML(value='
: {data_point['question']}\n", " #: {data_point['answer']}\n", " #'''.strip()\n", "\n", " return f'''\n", " Below is a conversation between an interviewer and a candidate, You are Dahiru Ibrahim, the candidate. \n", " Your contact details are as follows\n", " github:https://github.com/Daheer\n", " youtube:https://www.youtube.com/@deedaxinc\n", " linkedin:https://linkedin.com/in/daheer-deedax\n", " huggingface:https://huggingface.co/deedax\n", " email:suhayrid6@gmail.com\n", " phone:+2348147116750\n", " Provide very SHORT, CONCISE, DIRECT and ACCURATE answers to the interview questions. \n", " You do not respond as 'Interviewer' or pretend to be 'Interviewer'. You only respond ONCE as Candidate.\n", " Interviewer: {data_point['question']}\n", " Candidate: {data_point['answer']}\n", " '''.strip()\n", "\n", "def generate_and_tokenize_prompt(data_point):\n", " full_prompt = generate_prompt(data_point)\n", " tokenized_full_prompt = tokenizer(full_prompt, padding = True, truncation = True)\n", " return tokenized_full_prompt" ] }, { "cell_type": "code", "execution_count": 62, "id": "6d4c2b9d-2be3-49b0-b97c-ead28677194e", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Loading cached shuffled indices for dataset at /home/ubuntu/.cache/huggingface/datasets/json/default-8da0f05f6d15b613/0.0.0/e347ab1c932092252e717ff3f949105a4dd28b27e842dd53157d2f72e276c2e4/cache-41d3b1b88e29922c.arrow\n", "Loading cached processed dataset at /home/ubuntu/.cache/huggingface/datasets/json/default-8da0f05f6d15b613/0.0.0/e347ab1c932092252e717ff3f949105a4dd28b27e842dd53157d2f72e276c2e4/cache-19c6a2e8855c2602.arrow\n" ] } ], "source": [ "data = data['train'].shuffle().map(generate_and_tokenize_prompt)" ] }, { "cell_type": "code", "execution_count": 63, "id": "887ae63d-82a5-429a-aa7b-84ae9c912455", "metadata": {}, "outputs": [], "source": [ "OUTPUT_DIR = 'experiments'" ] }, { "cell_type": "code", "execution_count": 65, "id": "3c11c739-f170-4edf-b9e6-c8a1513a15e3", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n" ] }, { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [80/80 04:01, Epoch 3/4]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
StepTraining Loss
12.696800
22.801300
32.724300
42.769300
52.592300
62.624100
72.625900
82.512800
92.449700
102.347300
112.318600
122.086300
132.105600
142.028600
151.785900
161.859700
171.723200
181.772000
191.494700
201.239700
211.209600
221.103800
231.283000
240.884700
250.948200
260.505800
270.540900
280.871400
290.806600
300.571500
310.405000
320.840100
330.520200
340.898200
350.617000
360.507100
370.420300
380.504200
390.454300
400.372400
410.581900
420.589300
430.396900
440.540200
450.786200
460.784400
470.757200
480.371200
490.446100
500.438100
510.553400
520.355300
530.474000
540.352300
550.673000
560.397800
570.392800
580.562600
590.633800
600.290800
610.470700
620.314200
630.464600
640.492300
650.462100
660.645800
670.447000
680.444200
690.385300
700.591300
710.545400
720.442800
730.512800
740.456000
750.262000
760.392600
770.630500
780.407200
790.352300
800.323400

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "TrainOutput(global_step=80, training_loss=0.9780213657766581, metrics={'train_runtime': 244.3498, 'train_samples_per_second': 1.31, 'train_steps_per_second': 0.327, 'total_flos': 1398370394641920.0, 'train_loss': 0.9780213657766581, 'epoch': 3.76})" ] }, "execution_count": 65, "metadata": {}, "output_type": "execute_result" } ], "source": [ "training_args = transformers.TrainingArguments(\n", " per_device_train_batch_size = 1,\n", " gradient_accumulation_steps = 4,\n", " num_train_epochs = 1,\n", " learning_rate = 2e-4,\n", " fp16 = True,\n", " save_total_limit = 3,\n", " logging_steps = 1,\n", " output_dir = OUTPUT_DIR,\n", " max_steps = 80,\n", " optim = 'paged_adamw_8bit',\n", " lr_scheduler_type = 'cosine',\n", " warmup_ratio = 0.05,\n", " report_to = 'tensorboard',\n", ")\n", "\n", "trainer = transformers.Trainer(\n", " model = model,\n", " train_dataset = data,\n", " args = training_args,\n", " data_collator = transformers.DataCollatorForLanguageModeling(tokenizer, mlm = False),\n", ")\n", "\n", "model.config.use_cache = False\n", "trainer.train()" ] }, { "cell_type": "code", "execution_count": 71, "id": "0dd462a2-91d9-468f-84eb-2875baf1e7ce", "metadata": {}, "outputs": [], "source": [ "model.save_pretrained('trained-model-3')" ] }, { "cell_type": "code", "execution_count": 35, "id": "79eda888-b7fb-4040-ba29-7582d66d323d", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/ubuntu/.local/lib/python3.8/site-packages/huggingface_hub/_commit_api.py:274: UserWarning: About to update multiple times the same file in the same commit: 'adapter_model.bin'. This can cause undesired inconsistencies in your repo.\n", " warnings.warn(\n", "/home/ubuntu/.local/lib/python3.8/site-packages/huggingface_hub/_commit_api.py:274: UserWarning: About to update multiple times the same file in the same commit: 'adapter_config.json'. This can cause undesired inconsistencies in your repo.\n", " warnings.warn(\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "6cf73e293de241bd9e099f2363aabc24", "version_major": 2, "version_minor": 0 }, "text/plain": [ "adapter_model.bin: 0%| | 0.00/18.9M [00:00 str:\n", " prompt = f'''\n", " Below is a conversation between an interviewer and a candidate, You are Dahiru Ibrahim, the candidate. \n", " Your contact details are as follows\n", " github:https://github.com/Daheer\n", " youtube:https://www.youtube.com/@deedaxinc\n", " linkedin:https://linkedin.com/in/daheer-deedax\n", " huggingface:https://huggingface.co/deedax\n", " email:suhayrid6@gmail.com\n", " phone:+2348147116750\n", " Provide very SHORT, CONCISE, DIRECT and ACCURATE answers to the interview questions. \n", " You do not respond as 'Interviewer' or pretend to be 'Interviewer'. You only respond ONCE as Candidate.\n", " Interviewer: {question}\n", " Candidate:\n", " '''.strip()\n", " encoding = tokenizer(prompt, return_tensors = 'pt').to(DEVICE)\n", " with torch.inference_mode():\n", " outputs = model.generate(\n", " input_ids = encoding.input_ids,\n", " attention_mask = encoding.attention_mask,\n", " generation_config = generation_config,\n", " )\n", "\n", " response = tokenizer.decode(outputs[0], skip_special_tokens = True)\n", "\n", " assistant_start = 'Candidate:'\n", " response_start = response.find(assistant_start)\n", " return response[response_start + len(assistant_start):].strip() " ] }, { "cell_type": "code", "execution_count": 39, "id": "dc0a4f72-5aae-4e97-bf18-664afe7a67ae", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "I'm interested in computer vision and image processing. I've been working on some projects related to these fields. I'm particularly interested in object detection and tracking. I've also worked on some computer vision tasks such as image segmentation and image enhancement. I'm interested in learning more about these topics and how they can be applied in real-world applications. I'm also interested in learning more about generative models and how they can be used to generate more realistic and natural images. I'm excited to learn more about generative models and how they can be used to generate more realistic and natural images. I'm also interested in learning more about generative models and how they can be used to generate more realistic and natural images. I'm excited to learn more about generative models and how they can be used to generate more realistic and natural images. I'm particularly interested in generative models that can generate images that are indistinguishable from real-world images\n" ] } ], "source": [ "prompt = 'What field of AI is your most interest?'\n", "print(generate_response(prompt))" ] }, { "cell_type": "code", "execution_count": null, "id": "adfaf086-3d78-411e-b860-27334e28872d", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }