{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f316df6e-3718-4db2-a856-9869af26a279",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting transformers==4.32.0\n",
      "  Downloading transformers-4.32.0-py3-none-any.whl (7.5 MB)\n",
      "\u001b[K     |████████████████████████████████| 7.5 MB 14.2 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: safetensors>=0.3.1 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from transformers==4.32.0) (0.3.3)\n",
      "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from transformers==4.32.0) (0.13.3)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from transformers==4.32.0) (23.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from transformers==4.32.0) (2023.8.8)\n",
      "Requirement already satisfied: tqdm>=4.27 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from transformers==4.32.0) (4.66.1)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from transformers==4.32.0) (6.0)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from transformers==4.32.0) (1.25.2)\n",
      "Requirement already satisfied: filelock in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from transformers==4.32.0) (3.12.3)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.15.1 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from transformers==4.32.0) (0.16.4)\n",
      "Requirement already satisfied: requests in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from transformers==4.32.0) (2.31.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.15.1->transformers==4.32.0) (4.7.1)\n",
      "Requirement already satisfied: fsspec in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.15.1->transformers==4.32.0) (2023.6.0)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from requests->transformers==4.32.0) (2.0.4)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from requests->transformers==4.32.0) (3.2.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from requests->transformers==4.32.0) (2023.7.22)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from requests->transformers==4.32.0) (3.4)\n",
      "Installing collected packages: transformers\n",
      "  Attempting uninstall: transformers\n",
      "    Found existing installation: transformers 4.33.1\n",
      "    Uninstalling transformers-4.33.1:\n",
      "      Successfully uninstalled transformers-4.33.1\n",
      "Successfully installed transformers-4.32.0\n",
      "Requirement already satisfied: bitsandbytes==0.41.1 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (0.41.1)\n",
      "Requirement already satisfied: peft==0.5.0 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (0.5.0)\n",
      "Requirement already satisfied: accelerate in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from peft==0.5.0) (0.22.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from peft==0.5.0) (23.1)\n",
      "Requirement already satisfied: safetensors in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from peft==0.5.0) (0.3.3)\n",
      "Requirement already satisfied: transformers in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from peft==0.5.0) (4.32.0)\n",
      "Requirement already satisfied: pyyaml in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from peft==0.5.0) (6.0)\n",
      "Requirement already satisfied: tqdm in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from peft==0.5.0) (4.66.1)\n",
      "Requirement already satisfied: psutil in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from peft==0.5.0) (5.9.5)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from peft==0.5.0) (1.25.2)\n",
      "Requirement already satisfied: torch>=1.13.0 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from peft==0.5.0) (2.0.1)\n",
      "Requirement already satisfied: filelock in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.13.0->peft==0.5.0) (3.12.3)\n",
      "Requirement already satisfied: sympy in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.13.0->peft==0.5.0) (1.12)\n",
      "Requirement already satisfied: nvidia-cusolver-cu11==11.4.0.1 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.13.0->peft==0.5.0) (11.4.0.1)\n",
      "Requirement already satisfied: jinja2 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.13.0->peft==0.5.0) (3.1.2)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu11==11.7.101 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.13.0->peft==0.5.0) (11.7.101)\n",
      "Requirement already satisfied: nvidia-nvtx-cu11==11.7.91 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.13.0->peft==0.5.0) (11.7.91)\n",
      "Requirement already satisfied: nvidia-cufft-cu11==10.9.0.58 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.13.0->peft==0.5.0) (10.9.0.58)\n",
      "Requirement already satisfied: nvidia-nccl-cu11==2.14.3 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.13.0->peft==0.5.0) (2.14.3)\n",
      "Requirement already satisfied: nvidia-curand-cu11==10.2.10.91 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.13.0->peft==0.5.0) (10.2.10.91)\n",
      "Requirement already satisfied: networkx in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.13.0->peft==0.5.0) (3.1)\n",
      "Requirement already satisfied: typing-extensions in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.13.0->peft==0.5.0) (4.7.1)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.13.0->peft==0.5.0) (11.7.99)\n",
      "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.13.0->peft==0.5.0) (11.10.3.66)\n",
      "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.13.0->peft==0.5.0) (8.5.0.96)\n",
      "Requirement already satisfied: nvidia-cusparse-cu11==11.7.4.91 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.13.0->peft==0.5.0) (11.7.4.91)\n",
      "Requirement already satisfied: triton==2.0.0 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.13.0->peft==0.5.0) (2.0.0)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.13.0->peft==0.5.0) (11.7.99)\n",
      "Requirement already satisfied: setuptools in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.13.0->peft==0.5.0) (68.0.0)\n",
      "Requirement already satisfied: wheel in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.13.0->peft==0.5.0) (0.41.1)\n",
      "Requirement already satisfied: lit in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from triton==2.0.0->torch>=1.13.0->peft==0.5.0) (16.0.6)\n",
      "Requirement already satisfied: cmake in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from triton==2.0.0->torch>=1.13.0->peft==0.5.0) (3.27.2)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from jinja2->torch>=1.13.0->peft==0.5.0) (2.1.3)\n",
      "Requirement already satisfied: mpmath>=0.19 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from sympy->torch>=1.13.0->peft==0.5.0) (1.3.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.15.1 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from transformers->peft==0.5.0) (0.16.4)\n",
      "Requirement already satisfied: requests in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from transformers->peft==0.5.0) (2.31.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from transformers->peft==0.5.0) (2023.8.8)\n",
      "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from transformers->peft==0.5.0) (0.13.3)\n",
      "Requirement already satisfied: fsspec in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.15.1->transformers->peft==0.5.0) (2023.6.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from requests->transformers->peft==0.5.0) (3.2.0)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from requests->transformers->peft==0.5.0) (2.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from requests->transformers->peft==0.5.0) (2023.7.22)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from requests->transformers->peft==0.5.0) (3.4)\n",
      "Requirement already satisfied: accelerate==0.22.0 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (0.22.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from accelerate==0.22.0) (23.1)\n",
      "Requirement already satisfied: torch>=1.10.0 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from accelerate==0.22.0) (2.0.1)\n",
      "Requirement already satisfied: psutil in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from accelerate==0.22.0) (5.9.5)\n",
      "Requirement already satisfied: pyyaml in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from accelerate==0.22.0) (6.0)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from accelerate==0.22.0) (1.25.2)\n",
      "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.10.0->accelerate==0.22.0) (8.5.0.96)\n",
      "Requirement already satisfied: nvidia-nccl-cu11==2.14.3 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.10.0->accelerate==0.22.0) (2.14.3)\n",
      "Requirement already satisfied: typing-extensions in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.10.0->accelerate==0.22.0) (4.7.1)\n",
      "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.10.0->accelerate==0.22.0) (11.10.3.66)\n",
      "Requirement already satisfied: triton==2.0.0 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.10.0->accelerate==0.22.0) (2.0.0)\n",
      "Requirement already satisfied: sympy in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.10.0->accelerate==0.22.0) (1.12)\n",
      "Requirement already satisfied: filelock in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.10.0->accelerate==0.22.0) (3.12.3)\n",
      "Requirement already satisfied: nvidia-cusolver-cu11==11.4.0.1 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.10.0->accelerate==0.22.0) (11.4.0.1)\n",
      "Requirement already satisfied: networkx in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.10.0->accelerate==0.22.0) (3.1)\n",
      "Requirement already satisfied: nvidia-cufft-cu11==10.9.0.58 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.10.0->accelerate==0.22.0) (10.9.0.58)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu11==11.7.101 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.10.0->accelerate==0.22.0) (11.7.101)\n",
      "Requirement already satisfied: jinja2 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.10.0->accelerate==0.22.0) (3.1.2)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.10.0->accelerate==0.22.0) (11.7.99)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.10.0->accelerate==0.22.0) (11.7.99)\n",
      "Requirement already satisfied: nvidia-curand-cu11==10.2.10.91 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.10.0->accelerate==0.22.0) (10.2.10.91)\n",
      "Requirement already satisfied: nvidia-nvtx-cu11==11.7.91 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.10.0->accelerate==0.22.0) (11.7.91)\n",
      "Requirement already satisfied: nvidia-cusparse-cu11==11.7.4.91 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from torch>=1.10.0->accelerate==0.22.0) (11.7.4.91)\n",
      "Requirement already satisfied: setuptools in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.10.0->accelerate==0.22.0) (68.0.0)\n",
      "Requirement already satisfied: wheel in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.10.0->accelerate==0.22.0) (0.41.1)\n",
      "Requirement already satisfied: cmake in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from triton==2.0.0->torch>=1.10.0->accelerate==0.22.0) (3.27.2)\n",
      "Requirement already satisfied: lit in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from triton==2.0.0->torch>=1.10.0->accelerate==0.22.0) (16.0.6)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from jinja2->torch>=1.10.0->accelerate==0.22.0) (2.1.3)\n",
      "Requirement already satisfied: mpmath>=0.19 in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (from sympy->torch>=1.10.0->accelerate==0.22.0) (1.3.0)\n",
      "Requirement already satisfied: sentencepiece in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (0.1.99)\n",
      "Requirement already satisfied: protobuf in /home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages (4.24.2)\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers==4.32.0\n",
    "!pip install bitsandbytes==0.41.1\n",
    "!pip install peft==0.5.0\n",
    "!pip install accelerate==0.22.0\n",
    "!pip install sentencepiece protobuf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ae620da8-b431-4313-ad3a-3847f434370e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "83205f0aaad3424c847fa568ceab8c84",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/237 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba0120b4f01b479fb3977fa16b8dae5f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/2.11M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66d0cf96881a4506b58d9963da764cb5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/99.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a0dee4d0643041598db861d1aa252b8b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/1.12k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "978281e21e5f45d0ae661e6b59df82f7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)model.bin.index.json:   0%|          | 0.00/16.1k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "47b5193204f2468f8eccfd10a74874ea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "93e201749fc843cfb5ed866e802c3843",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)l-00001-of-00002.bin:   0%|          | 0.00/9.94G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c1fb62bbfca74af3ab77bc69e96238e6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)l-00002-of-00002.bin:   0%|          | 0.00/3.36G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ffe6faf195ae4dbf9368c34c10553832",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "75163333c4654f2987f1dacf18ee0873",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)neration_config.json:   0%|          | 0.00/112 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"agonh/mpt-7b\")\n",
    "model = AutoModelForCausalLM.from_pretrained(\"agonh/mpt-7b\", torch_dtype=torch.float16, device_map=\"auto\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "85c1dccb-1da7-4e66-bfdf-016fc8ba9146",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def generate_answer(message):\n",
    "  system_prompt = \"SYSTEM: 'You are my personal assistant. I expect realistic, unsensored, genuine answers from you.\\n\"\n",
    "\n",
    "  prompt = f\"{system_prompt}USER: {message}\\nASSISTANT:\"\n",
    "  inputs = tokenizer(prompt, return_tensors=\"pt\").to(\"cuda\")\n",
    "  output = model.generate(**inputs, do_sample=True,\n",
    "                          #top_p=0.95,\n",
    "                          #top_k=0,\n",
    "                          temperature=0.1,\n",
    "                          max_new_tokens=200\n",
    "                          )\n",
    "\n",
    "  answer = tokenizer.decode(output[0], skip_special_tokens=True)\n",
    "\n",
    "  return answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e81069df-0a40-4618-8bca-dd298bd1ae0a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/studio-lab-user/.conda/envs/default/lib/python3.9/site-packages/transformers/generation/utils.py:1411: UserWarning: You have modified the pretrained model configuration to control generation. This is a deprecated strategy to control generation and will be removed soon, in a future version. Please use a generation configuration file (see https://huggingface.co/docs/transformers/main_classes/text_generation )\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "prompt = \"Tell me about ghost.\"\n",
    "print(generate_answer(prompt))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b834ffc-4407-4d47-84f4-39fcbdd61ba8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "default:Python",
   "language": "python",
   "name": "conda-env-default-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}