{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3890292a-c99e-4367-955d-5883b93dba36",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: flash-attn in /opt/conda/lib/python3.10/site-packages (2.5.9.post1)\n",
      "Requirement already satisfied: torch in /opt/conda/lib/python3.10/site-packages (from flash-attn) (2.2.0)\n",
      "Requirement already satisfied: einops in /opt/conda/lib/python3.10/site-packages (from flash-attn) (0.8.0)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (3.13.1)\n",
      "Requirement already satisfied: typing-extensions>=4.8.0 in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (4.9.0)\n",
      "Requirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (1.12)\n",
      "Requirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (3.1)\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (3.1.2)\n",
      "Requirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (2023.12.2)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch->flash-attn) (2.1.3)\n",
      "Requirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch->flash-attn) (1.3.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install -q peft transformers datasets huggingface_hub\n",
    "!pip install flash-attn --no-build-isolation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f1cc378f-afb6-441f-a4c6-2ec427b4cd4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoModelForCausalLM, AutoTokenizer, default_data_collator, get_linear_schedule_with_warmup\n",
    "from peft import get_peft_config, get_peft_model, PromptTuningInit, PromptTuningConfig, TaskType, PeftType\n",
    "import torch\n",
    "from datasets import load_dataset\n",
    "import os\n",
    "from torch.utils.data import DataLoader\n",
    "from tqdm import tqdm\n",
    "from huggingface_hub import notebook_login\n",
    "from huggingface_hub import HfApi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4ab50d7-a4c9-4246-acd8-8875b87fe0da",
   "metadata": {},
   "outputs": [],
   "source": [
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "8a1cb1f9-b89d-4cac-a595-44e1e0ef85b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/Granther/prompt-tuned-phi3/commit/7ea57da9a4eccf3794c58bb4317df1c97a0fe2c8', commit_message='Upload prompt_tune_phi3.ipynb with huggingface_hub', commit_description='', oid='7ea57da9a4eccf3794c58bb4317df1c97a0fe2c8', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "api = HfApi()\n",
    "api.upload_file(path_or_fileobj='prompt_tune_phi3.ipynb',\n",
    "                path_in_repo='prompt_tune_phi3.ipynb',\n",
    "                repo_id='Granther/prompt-tuned-phi3',\n",
    "                repo_type='model'\n",
    "               )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "6cad1e5c-038f-4e75-8c3f-8ce0a43713a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = 'cuda'\n",
    "\n",
    "model_id = 'microsoft/Phi-3-mini-128k-instruct'\n",
    "\n",
    "peft_conf = PromptTuningConfig(\n",
    "    peft_type=PeftType.PROMPT_TUNING, # what kind of peft\n",
    "    task_type=TaskType.CAUSAL_LM,     # config task\n",
    "    prompt_tuning_init=PromptTuningInit.TEXT, # Set to 'TEXT' to use prompt_tuning_init_text\n",
    "    num_virtual_tokens=8, # x times the number of hidden transformer layers\n",
    "    prompt_tuning_init_text=\"Classify if the tweet is a complaint or not:\",\n",
    "    tokenizer_name_or_path=model_id\n",
    ")\n",
    "\n",
    "dataset_name = \"twitter_complaints\"\n",
    "checkpoint_name = f\"{dataset_name}_{model_id}_{peft_conf.peft_type}_{peft_conf.task_type}_v1.pt\".replace(\n",
    "    \"/\", \"_\"\n",
    ")\n",
    "\n",
    "text_col = 'Tweet text'\n",
    "label_col = 'text_label'\n",
    "max_len = 64\n",
    "lr = 3e-2\n",
    "epochs = 5\n",
    "batch_size = 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "6f677839-ef23-428a-bcfe-f596590804ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = load_dataset('ought/raft', dataset_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "c0c05613-7941-4959-ada9-49ed1093bec4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Unlabeled', 'complaint', 'no complaint']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset['train'].features['Label'].names\n",
    "#>>> ['Unlabeled', 'complaint', 'no complaint']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "14e2bc8b-b4e3-49c9-ae2b-5946e412caa5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "11da1eb81527428a95c41816f5bf459f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map (num_proc=10):   0%|          | 0/3399 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'Tweet text': '@HMRCcustomers No this is my first job',\n",
       " 'ID': 0,\n",
       " 'Label': 2,\n",
       " 'text_label': 'no complaint'}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create lambda function\n",
    "classes = [k.replace('_', ' ') for k in dataset['train'].features['Label'].names]\n",
    "dataset = dataset.map(\n",
    "    lambda x: {'text_label': [classes[label] for label in x['Label']]},\n",
    "    batched=True,\n",
    "    num_proc=10,\n",
    ")\n",
    "\n",
    "dataset['train'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "19f0865d-e490-4c9f-a5f4-e781ed270f47",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1, 853, 29880, 24025]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
    "\n",
    "if tokenizer.pad_token_id == None:\n",
    "    tokenizer.pad_token_id = tokenizer.eos_token_id\n",
    "\n",
    "target_max_len = max([len(tokenizer(class_lab)['input_ids']) for class_lab in classes])\n",
    "target_max_len # max length for tokenized labels\n",
    "\n",
    "tokenizer(classes[0])['input_ids'] \n",
    "# Ids corresponding to the tokens in the sequence\n",
    "# Attention mask is a binary tensor used in the transformer block to differentiate between padding tokens and meaningful ones"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1a15150-4bd9-45a2-ba43-d0bbbd16e60d",
   "metadata": {},
   "source": [
    "### Preprocess Function:\n",
    "- Tokenize text and label\n",
    "- Pad each example in the batch with tok.pad_token_id\n",
    "- "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "03f05467-dce3-4e42-ab3b-c39ba620e164",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preproc(example):\n",
    "    batch_size = len(example[text_col])\n",
    "    inputs = [f\"{text_col} : {x} Label : \" for x in example[text_col]]\n",
    "    # This is the text data that will be tokenized as the model input\n",
    "    targets = [str(x) for x in example[label_col]]\n",
    "    # Define batch of targets corresponding to inputs\n",
    "    model_inputs = tokenizer(inputs)\n",
    "    labels = tokenizer(targets)\n",
    "    # Tokenize\n",
    "\n",
    "    for i in range(batch_size):\n",
    "        sample_input_ids = model_inputs[\"input_ids\"][i]\n",
    "        label_input_ids = labels[\"input_ids\"][i] + [tokenizer.pad_token_id] # Appends to `input_ids` and not i\n",
    "\n",
    "        model_inputs[\"input_ids\"][i] = sample_input_ids + label_input_ids\n",
    "        # Afer tokenization, concatinate\n",
    "        labels[\"input_ids\"][i] = [-100] * len(sample_input_ids) + label_input_ids\n",
    "        #>>> -100, -100, -100, -100, -100, -100, -100, -100, 1, 694, 15313, 524, 32000\n",
    "        # Pad the beginning of the sequence with n -100s (ignore tokens)\n",
    "        model_inputs[\"attention_mask\"][i] = [1] * len(model_inputs[\"input_ids\"][i])\n",
    "\n",
    "    for i in range(batch_size):\n",
    "        sample_input_ids = model_inputs[\"input_ids\"][i]\n",
    "        label_input_ids = labels[\"input_ids\"][i]\n",
    "        model_inputs[\"input_ids\"][i] = [tokenizer.pad_token_id] * (target_max_len - len(sample_input_ids)) + sample_input_ids\n",
    "        model_inputs[\"attention_mask\"][i] = [0] * (target_max_len - len(sample_input_ids)) + model_inputs[\"attention_mask\"][i]\n",
    "        labels[\"input_ids\"][i] = [-100] * (target_max_len - len(sample_input_ids)) + label_input_ids\n",
    "        model_inputs[\"input_ids\"][i] = torch.tensor(model_inputs[\"input_ids\"][i][:target_max_len])\n",
    "        model_inputs[\"attention_mask\"][i] = torch.tensor(model_inputs[\"attention_mask\"][i][:target_max_len])\n",
    "        labels[\"input_ids\"][i] = torch.tensor(labels[\"input_ids\"][i][:target_max_len])\n",
    "        model_inputs[\"labels\"] = labels[\"input_ids\"]\n",
    "    return model_inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "72ddca5f-7bce-4342-9414-9dd9d41d9dec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05958c1cf67d413b9085622ace0cb799",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Preprocessing dataset (num_proc=10):   0%|          | 0/50 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05e7c3181c20464492f2ec4ced190fd4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Preprocessing dataset (num_proc=10):   0%|          | 0/3399 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "processed_datasets = dataset.map(\n",
    "    preproc,\n",
    "    batched=True, # uses default batch size\n",
    "    num_proc=10,\n",
    "    remove_columns=dataset[\"train\"].column_names, # All columns from the original dataset will be removed in the new dataset\n",
    "    load_from_cache_file=False,\n",
    "    desc=\"Preprocessing dataset\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "40cea6bc-e898-4d86-a6bf-5afc3a647e07",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = processed_datasets[\"train\"]\n",
    "eval_dataset = processed_datasets[\"test\"]\n",
    "\n",
    "train_dataloader = DataLoader(train_dataset,\n",
    "                              shuffle=True, # shuffling is unneccasary since we are not training\n",
    "                              collate_fn=default_data_collator,\n",
    "                              batch_size=batch_size,\n",
    "                              pin_memory=True # pin memory when using a GPU, makes loading data faster\n",
    "                             )\n",
    "\n",
    "eval_dataloader = DataLoader(eval_dataset,\n",
    "                              shuffle=False,\n",
    "                              collate_fn=default_data_collator,\n",
    "                              batch_size=batch_size,\n",
    "                              pin_memory=True\n",
    "                              )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "a4c529e4-d8ae-42b2-a658-f76d183bb264",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "58f2ef57b8ea49c2a26d4361ce4a5983",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable params: 24,576 || all params: 3,821,104,128 || trainable%: 0.0006\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = AutoModelForCausalLM.from_pretrained(model_id, attn_implementation=\"flash_attention_2\", torch_dtype=torch.bfloat16)\n",
    "model = get_peft_model(model, peft_conf)\n",
    "\n",
    "# the rest of the model is frozen\n",
    "print(model.print_trainable_parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "3289e4e3-9b9a-4256-921b-5df21d18344e",
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = torch.optim.AdamW(model.parameters(), lr=lr)\n",
    "lr_scheduler = get_linear_schedule_with_warmup(\n",
    "    optimizer=optimizer,\n",
    "    num_warmup_steps=0,\n",
    "    num_training_steps=(len(train_dataloader) * epochs),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "e7939d75-c6b9-47a8-b1a3-88f7c33ff121",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 7/7 [00:00<00:00, 10.97it/s]\n",
      "100%|██████████| 425/425 [00:13<00:00, 31.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch=0: train_ppl=tensor(nan, device='cuda:0') train_epoch_loss=tensor(nan, device='cuda:0') eval_ppl=tensor(nan, device='cuda:0') eval_epoch_loss=tensor(nan, device='cuda:0')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 7/7 [00:00<00:00, 12.02it/s]\n",
      "100%|██████████| 425/425 [00:13<00:00, 31.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch=1: train_ppl=tensor(nan, device='cuda:0') train_epoch_loss=tensor(nan, device='cuda:0') eval_ppl=tensor(nan, device='cuda:0') eval_epoch_loss=tensor(nan, device='cuda:0')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 7/7 [00:00<00:00, 12.70it/s]\n",
      "100%|██████████| 425/425 [00:13<00:00, 31.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch=2: train_ppl=tensor(nan, device='cuda:0') train_epoch_loss=tensor(nan, device='cuda:0') eval_ppl=tensor(nan, device='cuda:0') eval_epoch_loss=tensor(nan, device='cuda:0')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 7/7 [00:00<00:00, 11.85it/s]\n",
      "100%|██████████| 425/425 [00:13<00:00, 32.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch=3: train_ppl=tensor(nan, device='cuda:0') train_epoch_loss=tensor(nan, device='cuda:0') eval_ppl=tensor(nan, device='cuda:0') eval_epoch_loss=tensor(nan, device='cuda:0')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 7/7 [00:00<00:00, 12.53it/s]\n",
      "100%|██████████| 425/425 [00:13<00:00, 32.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch=4: train_ppl=tensor(nan, device='cuda:0') train_epoch_loss=tensor(nan, device='cuda:0') eval_ppl=tensor(nan, device='cuda:0') eval_epoch_loss=tensor(nan, device='cuda:0')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "model = model.to(device)\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    model.train()\n",
    "    total_loss = 0\n",
    "    for step, batch in enumerate(tqdm(train_dataloader)):\n",
    "        batch = {k: v.to(device) for k, v in batch.items()}\n",
    "        outputs = model(**batch)\n",
    "        loss = outputs.loss\n",
    "        total_loss += loss.detach().float()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        lr_scheduler.step()\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "    model.eval()\n",
    "    eval_loss = 0\n",
    "    eval_preds = []\n",
    "    for step, batch in enumerate(tqdm(eval_dataloader)):\n",
    "        batch = {k: v.to(device) for k, v in batch.items()}\n",
    "        with torch.no_grad():\n",
    "            outputs = model(**batch)\n",
    "        loss = outputs.loss\n",
    "        eval_loss += loss.detach().float()\n",
    "        eval_preds.extend(\n",
    "            tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True)\n",
    "        )\n",
    "\n",
    "    eval_epoch_loss = eval_loss / len(eval_dataloader)\n",
    "    eval_ppl = torch.exp(eval_epoch_loss)\n",
    "    train_epoch_loss = total_loss / len(train_dataloader)\n",
    "    train_ppl = torch.exp(train_epoch_loss)\n",
    "    print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "806d36f8-499e-4af8-b717-68e5d849866d",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_pretrained('model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "13db780a-fe20-4b23-b6cb-17118f7b695e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d8f94426025f4ad89847ac7e983cec42",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "device = 'cuda'\n",
    "pipe = pipeline(model='model', device=device, max_length=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "26438301-3601-44f4-bbe4-3c573a1c28be",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
      "You are not running the flash-attention implementation, expect numerical differences.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'generated_text': \"@nationalgridus I have no water and the bill is current and paid. Can you do something about this?\\n\\n### response\\nI understand your situation and I'm here to help. First, it's important to clarify that as an AI developed by Microsoft, I don't have the authority to directly intervene with your utility bills or the National Grid. However, I can guide you through the steps you should take to address this issue.\\n\\n1\"}]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe(\"@nationalgridus I have no water and the bill is current and paid. Can you do something about this?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f83e960d-ab80-406e-9ba9-e9533fe9d033",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}