init commit

.gitignore

@@ -0,0 +1,5 @@
+.ipynb_checkpoints
+autotrain/.ipynb_checkpoints
+.idea
+autotrain/autotrain_configs/*
+legacy

00-poe-generate-mistral-reasoning.ipynb

@@ -0,0 +1,1550 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "a51ae982-cad6-4d26-a18a-83472879325b",
+   "metadata": {},
+   "source": [
+    "# Setup"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "538e8d9b-74c8-4684-80be-623778bdaa90",
+   "metadata": {},
+   "source": [
+    "## Config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "cf368153-eb55-42f4-887f-889c4e9d7c60",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "INPUT_DATASET = 'layoric/labeled-multiple-choice-explained'\n",
+    "REVISION = '536f3b8'\n",
+    "OUTPUT_DATASET = 'derek-thomas/labeled-multiple-choice-explained-mistral-reasoning'\n",
+    "NAMESPACE = 'HF-test-lab' # Use your username or wherever you have the ability to deploy inference endpoints"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "c5a12bab-97bf-4a1d-989b-89f50a7fc272",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "aed670be03f741699118c33316dce7ec",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from transformers import AutoTokenizer\n",
+    "from huggingface_hub import get_token, login\n",
+    "\n",
+    "login()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "27c57c67-5574-4412-bde4-fae218e0f87f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "BASE_MODEL = 'mistralai/Mistral-7B-Instruct-v0.3'\n",
+    "tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, token=get_token())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ef2d177e-3651-4747-be3c-648be3a9b5ff",
+   "metadata": {},
+   "source": [
+    "# Mistral Prompt Creation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "447d6673-8bf5-4f56-aed5-d71bf47ac63f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from datasets import load_dataset\n",
+    "import json"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "08f3bf3d-cedf-4157-aba8-e1dc6d6edd08",
+   "metadata": {},
+   "source": [
+    "## Load and Cleanup Dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "bff8c5d2-9a80-4159-ae09-05ce70a53643",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load dataset from Hugging Face Hub\n",
+    "dataset = load_dataset(INPUT_DATASET, split='train', revision=REVISION)\n",
+    "\n",
+    "# Convert to pandas dataframe\n",
+    "df = dataset.to_pandas()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "3119c978-9e2e-45f3-8389-d61d6733dbdb",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>formatted_question</th>\n",
+       "      <th>combinedfact</th>\n",
+       "      <th>answerKey</th>\n",
+       "      <th>topic</th>\n",
+       "      <th>__index_level_0__</th>\n",
+       "      <th>explanation</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>what is satellite technology used for predicti...</td>\n",
+       "      <td>satellite technology is used for predicting wh...</td>\n",
+       "      <td>c</td>\n",
+       "      <td>technology</td>\n",
+       "      <td>35972.0</td>\n",
+       "      <td>a) Seconds and minutes: This option is incorre...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>what does irradiating food do? (a) relieve pai...</td>\n",
+       "      <td>irradiated food improves food safety.</td>\n",
+       "      <td>c</td>\n",
+       "      <td>food science</td>\n",
+       "      <td>20412.0</td>\n",
+       "      <td>(a) Relieve pain: This option is not correct b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>what protects a mammal's skin? (a) fiber folli...</td>\n",
+       "      <td>fiber follicles protect mammal skin</td>\n",
+       "      <td>a</td>\n",
+       "      <td>biology</td>\n",
+       "      <td>5779.0</td>\n",
+       "      <td>b) Exfoliation: Exfoliation is the process of ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>what do earthworms do when a segment breaks of...</td>\n",
+       "      <td>earthworms can regrow segments that break off</td>\n",
+       "      <td>b</td>\n",
+       "      <td>biology</td>\n",
+       "      <td>4064.0</td>\n",
+       "      <td>a) Dies: This option is not correct because ea...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>lightning can be bad for what? (a) the environ...</td>\n",
+       "      <td>lightning can be bad for the environment.</td>\n",
+       "      <td>a</td>\n",
+       "      <td>electricity</td>\n",
+       "      <td>34080.0</td>\n",
+       "      <td>b) Rainstorms: Lightning is actually a natural...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9093</th>\n",
+       "      <td>organisms that can cause infection do what? (a...</td>\n",
+       "      <td>organisms that can cause infection make humans...</td>\n",
+       "      <td>g</td>\n",
+       "      <td>biology</td>\n",
+       "      <td>33485.0</td>\n",
+       "      <td>a) Bandaging open sores is not the correct ans...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9094</th>\n",
+       "      <td>fungi are living things that cannot make thei...</td>\n",
+       "      <td>fungi are living things that cannot make their...</td>\n",
+       "      <td>a</td>\n",
+       "      <td>biology</td>\n",
+       "      <td>12097.0</td>\n",
+       "      <td>b) Fungi are living things that can make their...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9095</th>\n",
+       "      <td>an overheated body can use water for: (a) meta...</td>\n",
+       "      <td>the evaporation of water from the skin cools t...</td>\n",
+       "      <td>g</td>\n",
+       "      <td>biology</td>\n",
+       "      <td>6522.0</td>\n",
+       "      <td>a) Metabolic reaction: This option is incorrec...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9096</th>\n",
+       "      <td>what is essential for cellular respiration for...</td>\n",
+       "      <td>plants are essential for cellular respiration ...</td>\n",
+       "      <td>f</td>\n",
+       "      <td>biology</td>\n",
+       "      <td>27144.0</td>\n",
+       "      <td>a) Electrons are involved in cellular respirat...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9097</th>\n",
+       "      <td>what helps insulate and protect the body? (a) ...</td>\n",
+       "      <td>living cells in follicles help insulate and pr...</td>\n",
+       "      <td>b</td>\n",
+       "      <td>biology</td>\n",
+       "      <td>18522.0</td>\n",
+       "      <td>a) H2O: Water is essential for life, but it do...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>9098 rows × 6 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                     formatted_question  \\\n",
+       "0     what is satellite technology used for predicti...   \n",
+       "1     what does irradiating food do? (a) relieve pai...   \n",
+       "2     what protects a mammal's skin? (a) fiber folli...   \n",
+       "3     what do earthworms do when a segment breaks of...   \n",
+       "4     lightning can be bad for what? (a) the environ...   \n",
+       "...                                                 ...   \n",
+       "9093  organisms that can cause infection do what? (a...   \n",
+       "9094   fungi are living things that cannot make thei...   \n",
+       "9095  an overheated body can use water for: (a) meta...   \n",
+       "9096  what is essential for cellular respiration for...   \n",
+       "9097  what helps insulate and protect the body? (a) ...   \n",
+       "\n",
+       "                                           combinedfact answerKey  \\\n",
+       "0     satellite technology is used for predicting wh...         c   \n",
+       "1                 irradiated food improves food safety.         c   \n",
+       "2                   fiber follicles protect mammal skin         a   \n",
+       "3         earthworms can regrow segments that break off         b   \n",
+       "4             lightning can be bad for the environment.         a   \n",
+       "...                                                 ...       ...   \n",
+       "9093  organisms that can cause infection make humans...         g   \n",
+       "9094  fungi are living things that cannot make their...         a   \n",
+       "9095  the evaporation of water from the skin cools t...         g   \n",
+       "9096  plants are essential for cellular respiration ...         f   \n",
+       "9097  living cells in follicles help insulate and pr...         b   \n",
+       "\n",
+       "             topic  __index_level_0__  \\\n",
+       "0       technology            35972.0   \n",
+       "1     food science            20412.0   \n",
+       "2          biology             5779.0   \n",
+       "3          biology             4064.0   \n",
+       "4      electricity            34080.0   \n",
+       "...            ...                ...   \n",
+       "9093       biology            33485.0   \n",
+       "9094       biology            12097.0   \n",
+       "9095       biology             6522.0   \n",
+       "9096       biology            27144.0   \n",
+       "9097       biology            18522.0   \n",
+       "\n",
+       "                                            explanation  \n",
+       "0     a) Seconds and minutes: This option is incorre...  \n",
+       "1     (a) Relieve pain: This option is not correct b...  \n",
+       "2     b) Exfoliation: Exfoliation is the process of ...  \n",
+       "3     a) Dies: This option is not correct because ea...  \n",
+       "4     b) Rainstorms: Lightning is actually a natural...  \n",
+       "...                                                 ...  \n",
+       "9093  a) Bandaging open sores is not the correct ans...  \n",
+       "9094  b) Fungi are living things that can make their...  \n",
+       "9095  a) Metabolic reaction: This option is incorrec...  \n",
+       "9096  a) Electrons are involved in cellular respirat...  \n",
+       "9097  a) H2O: Water is essential for life, but it do...  \n",
+       "\n",
+       "[9098 rows x 6 columns]"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "a1ec5678-13a5-4602-9d67-8406248fa414",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Before Cleaning: 9098 rows\n",
+      "After Cleaning: 8413 rows\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(f\"Before Cleaning: {len(df)} rows\")\n",
+    "\n",
+    "# Drop the __index_level_0__ column if it exists\n",
+    "df.drop(columns=['__index_level_0__'], errors='ignore', inplace=True)\n",
+    "\n",
+    "# Ensure all values in 'formatted_question' are strings\n",
+    "df['formatted_question'] = df['formatted_question'].astype(str)\n",
+    "\n",
+    "# Filter on topic so that we can stratify later\n",
+    "df['topic'] = df['topic'].fillna('unknown')\n",
+    "topic_counts = df['topic'].value_counts()\n",
+    "valid_topics = topic_counts[topic_counts >= 5].index\n",
+    "df = df[df['topic'].isin(valid_topics)]\n",
+    "df = df[~df['topic'].isin(['unknown'])]\n",
+    "\n",
+    "df.rename(columns={\n",
+    "    'combinedfact': 'combined_fact',\n",
+    "    'answerKey': 'answer_key' \n",
+    "}, inplace=True)\n",
+    "\n",
+    "\n",
+    "# Drop the index level 0 if it exists\n",
+    "df.reset_index(drop=True, inplace=True)\n",
+    "print(f\"After Cleaning: {len(df)} rows\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "71337823-91b9-4946-95cd-71810f5c5101",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>formatted_question</th>\n",
+       "      <th>combinedfact</th>\n",
+       "      <th>answer_key</th>\n",
+       "      <th>topic</th>\n",
+       "      <th>explanation</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>what is satellite technology used for predicti...</td>\n",
+       "      <td>satellite technology is used for predicting wh...</td>\n",
+       "      <td>c</td>\n",
+       "      <td>technology</td>\n",
+       "      <td>a) Seconds and minutes: This option is incorre...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>what does irradiating food do? (a) relieve pai...</td>\n",
+       "      <td>irradiated food improves food safety.</td>\n",
+       "      <td>c</td>\n",
+       "      <td>food science</td>\n",
+       "      <td>(a) Relieve pain: This option is not correct b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>what protects a mammal's skin? (a) fiber folli...</td>\n",
+       "      <td>fiber follicles protect mammal skin</td>\n",
+       "      <td>a</td>\n",
+       "      <td>biology</td>\n",
+       "      <td>b) Exfoliation: Exfoliation is the process of ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>what do earthworms do when a segment breaks of...</td>\n",
+       "      <td>earthworms can regrow segments that break off</td>\n",
+       "      <td>b</td>\n",
+       "      <td>biology</td>\n",
+       "      <td>a) Dies: This option is not correct because ea...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>lightning can be bad for what? (a) the environ...</td>\n",
+       "      <td>lightning can be bad for the environment.</td>\n",
+       "      <td>a</td>\n",
+       "      <td>electricity</td>\n",
+       "      <td>b) Rainstorms: Lightning is actually a natural...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8408</th>\n",
+       "      <td>organisms that can cause infection do what? (a...</td>\n",
+       "      <td>organisms that can cause infection make humans...</td>\n",
+       "      <td>g</td>\n",
+       "      <td>biology</td>\n",
+       "      <td>a) Bandaging open sores is not the correct ans...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8409</th>\n",
+       "      <td>fungi are living things that cannot make thei...</td>\n",
+       "      <td>fungi are living things that cannot make their...</td>\n",
+       "      <td>a</td>\n",
+       "      <td>biology</td>\n",
+       "      <td>b) Fungi are living things that can make their...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8410</th>\n",
+       "      <td>an overheated body can use water for: (a) meta...</td>\n",
+       "      <td>the evaporation of water from the skin cools t...</td>\n",
+       "      <td>g</td>\n",
+       "      <td>biology</td>\n",
+       "      <td>a) Metabolic reaction: This option is incorrec...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8411</th>\n",
+       "      <td>what is essential for cellular respiration for...</td>\n",
+       "      <td>plants are essential for cellular respiration ...</td>\n",
+       "      <td>f</td>\n",
+       "      <td>biology</td>\n",
+       "      <td>a) Electrons are involved in cellular respirat...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8412</th>\n",
+       "      <td>what helps insulate and protect the body? (a) ...</td>\n",
+       "      <td>living cells in follicles help insulate and pr...</td>\n",
+       "      <td>b</td>\n",
+       "      <td>biology</td>\n",
+       "      <td>a) H2O: Water is essential for life, but it do...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>8413 rows × 5 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                     formatted_question  \\\n",
+       "0     what is satellite technology used for predicti...   \n",
+       "1     what does irradiating food do? (a) relieve pai...   \n",
+       "2     what protects a mammal's skin? (a) fiber folli...   \n",
+       "3     what do earthworms do when a segment breaks of...   \n",
+       "4     lightning can be bad for what? (a) the environ...   \n",
+       "...                                                 ...   \n",
+       "8408  organisms that can cause infection do what? (a...   \n",
+       "8409   fungi are living things that cannot make thei...   \n",
+       "8410  an overheated body can use water for: (a) meta...   \n",
+       "8411  what is essential for cellular respiration for...   \n",
+       "8412  what helps insulate and protect the body? (a) ...   \n",
+       "\n",
+       "                                           combinedfact answer_key  \\\n",
+       "0     satellite technology is used for predicting wh...          c   \n",
+       "1                 irradiated food improves food safety.          c   \n",
+       "2                   fiber follicles protect mammal skin          a   \n",
+       "3         earthworms can regrow segments that break off          b   \n",
+       "4             lightning can be bad for the environment.          a   \n",
+       "...                                                 ...        ...   \n",
+       "8408  organisms that can cause infection make humans...          g   \n",
+       "8409  fungi are living things that cannot make their...          a   \n",
+       "8410  the evaporation of water from the skin cools t...          g   \n",
+       "8411  plants are essential for cellular respiration ...          f   \n",
+       "8412  living cells in follicles help insulate and pr...          b   \n",
+       "\n",
+       "             topic                                        explanation  \n",
+       "0       technology  a) Seconds and minutes: This option is incorre...  \n",
+       "1     food science  (a) Relieve pain: This option is not correct b...  \n",
+       "2          biology  b) Exfoliation: Exfoliation is the process of ...  \n",
+       "3          biology  a) Dies: This option is not correct because ea...  \n",
+       "4      electricity  b) Rainstorms: Lightning is actually a natural...  \n",
+       "...            ...                                                ...  \n",
+       "8408       biology  a) Bandaging open sores is not the correct ans...  \n",
+       "8409       biology  b) Fungi are living things that can make their...  \n",
+       "8410       biology  a) Metabolic reaction: This option is incorrec...  \n",
+       "8411       biology  a) Electrons are involved in cellular respirat...  \n",
+       "8412       biology  a) H2O: Water is essential for life, but it do...  \n",
+       "\n",
+       "[8413 rows x 5 columns]"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "eb6efb48-1282-4539-a726-450a17fd7f12",
+   "metadata": {},
+   "source": [
+    "## Process and Format Questions and Answer Choices\n",
+    "I do some minor cleaning since the original dataset is a bit messy."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "c3ac53c9-664a-4d8e-9820-7be788d120fa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Split before the first occurrence of \"(a)\" to separate question and answer choices\n",
+    "df['topic'] = df['topic'].str.capitalize()\n",
+    "df['question_text'] = df['formatted_question'].str.extract(r'^(.*?)(?=\\(a\\)|$)')[0]\n",
+    "df['answer_choices'] = df['formatted_question'].str.extract(r'(\\(a\\).*)$')[0]\n",
+    "\n",
+    "# Function to check if the text needs a question mark\n",
+    "def add_question_mark_if_needed(text):\n",
+    "    text = text.strip().capitalize()\n",
+    "    if not text.endswith('?'):\n",
+    "        return text + '?'\n",
+    "    return text\n",
+    "\n",
+    "# Apply the function to the 'question_text' column\n",
+    "df['question_text'] = df['question_text'].apply(add_question_mark_if_needed)\n",
+    "\n",
+    "# Function to capitalize each answer choice item\n",
+    "def capitalize_answer_choices(answer_choices):\n",
+    "    import re\n",
+    "    pattern = r'\\(([a-z])\\)\\s*([^\\(]+)'\n",
+    "    matches = re.findall(pattern, answer_choices)\n",
+    "    capitalized_choices = []\n",
+    "    for match in matches:\n",
+    "        label, text = match\n",
+    "        capitalized_choices.append(f'({label}) {text.strip().capitalize()}')\n",
+    "    return ' '.join(capitalized_choices)\n",
+    "\n",
+    "# Apply function to format answer choices\n",
+    "df['answer_choices'] = df['answer_choices'].apply(capitalize_answer_choices)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "f96ed8a7-77d2-4ee4-b3a1-7dcb30f5966e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>formatted_question</th>\n",
+       "      <th>combinedfact</th>\n",
+       "      <th>answer_key</th>\n",
+       "      <th>topic</th>\n",
+       "      <th>explanation</th>\n",
+       "      <th>question_text</th>\n",
+       "      <th>answer_choices</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>what is satellite technology used for predicti...</td>\n",
+       "      <td>satellite technology is used for predicting wh...</td>\n",
+       "      <td>c</td>\n",
+       "      <td>Technology</td>\n",
+       "      <td>a) Seconds and minutes: This option is incorre...</td>\n",
+       "      <td>What is satellite technology used for predicting?</td>\n",
+       "      <td>(a) Seconds and minutes (b) The strength and m...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>what does irradiating food do? (a) relieve pai...</td>\n",
+       "      <td>irradiated food improves food safety.</td>\n",
+       "      <td>c</td>\n",
+       "      <td>Food science</td>\n",
+       "      <td>(a) Relieve pain: This option is not correct b...</td>\n",
+       "      <td>What does irradiating food do?</td>\n",
+       "      <td>(a) Relieve pain (b) Enhance food's nutrients ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>what protects a mammal's skin? (a) fiber folli...</td>\n",
+       "      <td>fiber follicles protect mammal skin</td>\n",
+       "      <td>a</td>\n",
+       "      <td>Biology</td>\n",
+       "      <td>b) Exfoliation: Exfoliation is the process of ...</td>\n",
+       "      <td>What protects a mammal's skin?</td>\n",
+       "      <td>(a) Fiber follicles (b) Exfoliation (c) Resist...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>what do earthworms do when a segment breaks of...</td>\n",
+       "      <td>earthworms can regrow segments that break off</td>\n",
+       "      <td>b</td>\n",
+       "      <td>Biology</td>\n",
+       "      <td>a) Dies: This option is not correct because ea...</td>\n",
+       "      <td>What do earthworms do when a segment breaks off?</td>\n",
+       "      <td>(a) Dies (b) Regrows it (c) Reproduces (d) Sed...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>lightning can be bad for what? (a) the environ...</td>\n",
+       "      <td>lightning can be bad for the environment.</td>\n",
+       "      <td>a</td>\n",
+       "      <td>Electricity</td>\n",
+       "      <td>b) Rainstorms: Lightning is actually a natural...</td>\n",
+       "      <td>Lightning can be bad for what?</td>\n",
+       "      <td>(a) The environment (b) Rainstorms (c) Destruc...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8408</th>\n",
+       "      <td>organisms that can cause infection do what? (a...</td>\n",
+       "      <td>organisms that can cause infection make humans...</td>\n",
+       "      <td>g</td>\n",
+       "      <td>Biology</td>\n",
+       "      <td>a) Bandaging open sores is not the correct ans...</td>\n",
+       "      <td>Organisms that can cause infection do what?</td>\n",
+       "      <td>(a) Bandage open sores (b) Keep flesh clean (c...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8409</th>\n",
+       "      <td>fungi are living things that cannot make thei...</td>\n",
+       "      <td>fungi are living things that cannot make their...</td>\n",
+       "      <td>a</td>\n",
+       "      <td>Biology</td>\n",
+       "      <td>b) Fungi are living things that can make their...</td>\n",
+       "      <td>Fungi are living things that cannot make their...</td>\n",
+       "      <td>(a) Food (b) Cells (c) Energy (d) Fruits (e) H...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8410</th>\n",
+       "      <td>an overheated body can use water for: (a) meta...</td>\n",
+       "      <td>the evaporation of water from the skin cools t...</td>\n",
+       "      <td>g</td>\n",
+       "      <td>Biology</td>\n",
+       "      <td>a) Metabolic reaction: This option is incorrec...</td>\n",
+       "      <td>An overheated body can use water for:?</td>\n",
+       "      <td>(a) Metabolic reaction (b) Dehydrating (c) Rai...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8411</th>\n",
+       "      <td>what is essential for cellular respiration for...</td>\n",
+       "      <td>plants are essential for cellular respiration ...</td>\n",
+       "      <td>f</td>\n",
+       "      <td>Biology</td>\n",
+       "      <td>a) Electrons are involved in cellular respirat...</td>\n",
+       "      <td>What is essential for cellular respiration for...</td>\n",
+       "      <td>(a) Electron (b) Glucose (c) Energy (d) Energy...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8412</th>\n",
+       "      <td>what helps insulate and protect the body? (a) ...</td>\n",
+       "      <td>living cells in follicles help insulate and pr...</td>\n",
+       "      <td>b</td>\n",
+       "      <td>Biology</td>\n",
+       "      <td>a) H2O: Water is essential for life, but it do...</td>\n",
+       "      <td>What helps insulate and protect the body?</td>\n",
+       "      <td>(a) H2o (b) Living cells in follicles (c) Laye...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>8413 rows × 7 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                     formatted_question  \\\n",
+       "0     what is satellite technology used for predicti...   \n",
+       "1     what does irradiating food do? (a) relieve pai...   \n",
+       "2     what protects a mammal's skin? (a) fiber folli...   \n",
+       "3     what do earthworms do when a segment breaks of...   \n",
+       "4     lightning can be bad for what? (a) the environ...   \n",
+       "...                                                 ...   \n",
+       "8408  organisms that can cause infection do what? (a...   \n",
+       "8409   fungi are living things that cannot make thei...   \n",
+       "8410  an overheated body can use water for: (a) meta...   \n",
+       "8411  what is essential for cellular respiration for...   \n",
+       "8412  what helps insulate and protect the body? (a) ...   \n",
+       "\n",
+       "                                           combinedfact answer_key  \\\n",
+       "0     satellite technology is used for predicting wh...          c   \n",
+       "1                 irradiated food improves food safety.          c   \n",
+       "2                   fiber follicles protect mammal skin          a   \n",
+       "3         earthworms can regrow segments that break off          b   \n",
+       "4             lightning can be bad for the environment.          a   \n",
+       "...                                                 ...        ...   \n",
+       "8408  organisms that can cause infection make humans...          g   \n",
+       "8409  fungi are living things that cannot make their...          a   \n",
+       "8410  the evaporation of water from the skin cools t...          g   \n",
+       "8411  plants are essential for cellular respiration ...          f   \n",
+       "8412  living cells in follicles help insulate and pr...          b   \n",
+       "\n",
+       "             topic                                        explanation  \\\n",
+       "0       Technology  a) Seconds and minutes: This option is incorre...   \n",
+       "1     Food science  (a) Relieve pain: This option is not correct b...   \n",
+       "2          Biology  b) Exfoliation: Exfoliation is the process of ...   \n",
+       "3          Biology  a) Dies: This option is not correct because ea...   \n",
+       "4      Electricity  b) Rainstorms: Lightning is actually a natural...   \n",
+       "...            ...                                                ...   \n",
+       "8408       Biology  a) Bandaging open sores is not the correct ans...   \n",
+       "8409       Biology  b) Fungi are living things that can make their...   \n",
+       "8410       Biology  a) Metabolic reaction: This option is incorrec...   \n",
+       "8411       Biology  a) Electrons are involved in cellular respirat...   \n",
+       "8412       Biology  a) H2O: Water is essential for life, but it do...   \n",
+       "\n",
+       "                                          question_text  \\\n",
+       "0     What is satellite technology used for predicting?   \n",
+       "1                        What does irradiating food do?   \n",
+       "2                        What protects a mammal's skin?   \n",
+       "3      What do earthworms do when a segment breaks off?   \n",
+       "4                        Lightning can be bad for what?   \n",
+       "...                                                 ...   \n",
+       "8408        Organisms that can cause infection do what?   \n",
+       "8409  Fungi are living things that cannot make their...   \n",
+       "8410             An overheated body can use water for:?   \n",
+       "8411  What is essential for cellular respiration for...   \n",
+       "8412          What helps insulate and protect the body?   \n",
+       "\n",
+       "                                         answer_choices  \n",
+       "0     (a) Seconds and minutes (b) The strength and m...  \n",
+       "1     (a) Relieve pain (b) Enhance food's nutrients ...  \n",
+       "2     (a) Fiber follicles (b) Exfoliation (c) Resist...  \n",
+       "3     (a) Dies (b) Regrows it (c) Reproduces (d) Sed...  \n",
+       "4     (a) The environment (b) Rainstorms (c) Destruc...  \n",
+       "...                                                 ...  \n",
+       "8408  (a) Bandage open sores (b) Keep flesh clean (c...  \n",
+       "8409  (a) Food (b) Cells (c) Energy (d) Fruits (e) H...  \n",
+       "8410  (a) Metabolic reaction (b) Dehydrating (c) Rai...  \n",
+       "8411  (a) Electron (b) Glucose (c) Energy (d) Energy...  \n",
+       "8412  (a) H2o (b) Living cells in follicles (c) Laye...  \n",
+       "\n",
+       "[8413 rows x 7 columns]"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8d911d0a-2097-4d16-8bd8-c1527ebd20ca",
+   "metadata": {},
+   "source": [
+    "## Prompt Creation"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "524aa429-fec5-4ba2-add7-e27b70d44854",
+   "metadata": {},
+   "source": [
+    "This is the original prompt from layoric. It looks pretty good, but we need to adjust this for mistral. \n",
+    "```\n",
+    "{{#role 'system'~}}\n",
+    "You are an AI assistant that helps people find information. User will give you a question. Your task is to answer as faithfully as you can, and most importantly, provide explanation why incorrect answers are not correct. While answering think step-by-step and justify your answer.\n",
+    "{{~/role}}\n",
+    "{{#role 'user'~}}\n",
+    "USER:\n",
+    "Topic: {{topic}}\n",
+    "Question: {{question}}\n",
+    "\n",
+    "### Answer\n",
+    "The correct answer is:\n",
+    "{{answer_key}}). {{answer}}\n",
+    "\n",
+    "### Explanation:\n",
+    "Let's break it down step by step.\n",
+    "\n",
+    "1. Read the question and options carefully.\n",
+    "2. Identify the differences between the options.\n",
+    "3. Determine which options are not logical based on the difference.\n",
+    "4. Go through each incorrect answer providing an explanation why it is incorrect.\n",
+    "{{~/role}}\n",
+    "\n",
+    "{{#role 'assistant'~}}\n",
+    "{{~gen 'explanation'}}\n",
+    "{{~/role}}\n",
+    "```\n",
+    "Changes:\n",
+    "1. We need to move the system instructions to the user message\n",
+    "2. Let's rename `explanation` to `reasoning`\n",
+    "\n",
+    "We want something like this:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "0e6b3afc-8f53-490b-92ca-aec1216f62cc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def create_mistral_reasoning_prompt(row):\n",
+    "    reasoning_instructions = f'''You are an AI assistant that helps people find information. User will give you a question and an answer. Your task is to explain your reasoning of the correct answer and provide explanation why incorrect answers are not correct. While answering, think step-by-step and justify your answer.\n",
+    "\n",
+    "Topic: {row['topic']}\n",
+    "Question: {row['question_text']}\n",
+    "Answer Choices: {row['answer_choices']}\n",
+    "\n",
+    "The correct answer is: ({row['answer_key']})\n",
+    "\n",
+    "### Reasoning:\n",
+    "Let's break it down step by step.\n",
+    "\n",
+    "1. Read the question and options carefully.\n",
+    "2. Identify the differences between the options.\n",
+    "3. Determine which options are not logical based on the difference.\n",
+    "4. Go through each incorrect answer providing an explanation why it is incorrect.'''\n",
+    "    \n",
+    "    return [{\"role\": \"user\", \"content\": reasoning_instructions}]\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3e759e97-c046-4fd1-a924-7618d1756588",
+   "metadata": {},
+   "source": [
+    "Note that I updated the instructions slightly."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "1e27e8b4-5cb3-42bf-8b2f-d0627c1d1ae9",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[{'role': 'user', 'content': \"You are an AI assistant that helps people find information. User will give you a question and an answer. Your task is to explain your reasoning of the correct answer and provide explanation why incorrect answers are not correct. While answering, think step-by-step and justify your answer.\\n\\nTopic: Technology\\nQuestion: What is satellite technology used for predicting?\\nAnswer Choices: (a) Seconds and minutes (b) The strength and magnitude of an earthquake (c) What it's like outside each day (d) 70-75 degrees fahrenheit (e) Rapid changes occur (f) Dead-ends and false starts. (g) Snow, ice, and rock (h) Around 5 to 27 degrees celsius\\n\\nThe correct answer is: (c)\\n\\n### Reasoning:\\nLet's break it down step by step.\\n\\n1. Read the question and options carefully.\\n2. Identify the differences between the options.\\n3. Determine which options are not logical based on the difference.\\n4. Go through each incorrect answer providing an explanation why it is incorrect.\"}]\n",
+      "\n",
+      "---\n",
+      "\n",
+      "You are an AI assistant that helps people find information. User will give you a question and an answer. Your task is to explain your reasoning of the correct answer and provide explanation why incorrect answers are not correct. While answering, think step-by-step and justify your answer.\n",
+      "\n",
+      "Topic: Technology\n",
+      "Question: What is satellite technology used for predicting?\n",
+      "Answer Choices: (a) Seconds and minutes (b) The strength and magnitude of an earthquake (c) What it's like outside each day (d) 70-75 degrees fahrenheit (e) Rapid changes occur (f) Dead-ends and false starts. (g) Snow, ice, and rock (h) Around 5 to 27 degrees celsius\n",
+      "\n",
+      "The correct answer is: (c)\n",
+      "\n",
+      "### Reasoning:\n",
+      "Let's break it down step by step.\n",
+      "\n",
+      "1. Read the question and options carefully.\n",
+      "2. Identify the differences between the options.\n",
+      "3. Determine which options are not logical based on the difference.\n",
+      "4. Go through each incorrect answer providing an explanation why it is incorrect.\n"
+     ]
+    }
+   ],
+   "source": [
+    "df['mistral_reasoning_prompt'] = df.apply(create_mistral_reasoning_prompt, axis=1)\n",
+    "print(df['mistral_reasoning_prompt'].iloc[0])\n",
+    "print('\\n---\\n')\n",
+    "print(df['mistral_reasoning_prompt'].iloc[0][0]['content'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "1f9c248e-fef9-4194-a567-0ee10be7184d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAHHCAYAAABeLEexAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABDaElEQVR4nO3deVxV1d7H8e9BmRwAEQG5Kmqa83AzB3JOE6dyqnudUsuyATTTrLx5U7OyNIcG03zK6TZodrN8nBJnU7M0yRmHLCxBQwUEFRHW80cv9tMRVMTDoPvzfr32q/baa6/z2yuFb3s422GMMQIAALAxt8IuAAAAoLARiAAAgO0RiAAAgO0RiAAAgO0RiAAAgO0RiAAAgO0RiAAAgO0RiAAAgO0RiAAAgO0RiIBCVrlyZQ0aNKiwy7jtTZ48WVWrVlWxYsXUsGHDfP2sDRs2yOFw6IsvvsjXzwHgOgQiwIXmzZsnh8OhHTt25Li9TZs2qlu37k1/zooVKzRu3LibHscuVq9ereeff17NmzfX3Llz9frrr2frkxVicrPcii5evKhp06apadOm8vX1lZeXl+68805FRkbq0KFDhV2eJGnr1q0aN26cEhMTC7sU2FDxwi4AsLuYmBi5ud3Y/5usWLFCM2bMIBTl0rp16+Tm5qaPPvpIHh4eOfapVauW/vOf/zi1jR49WqVKldJLL71UEGXmm4SEBHXs2FE7d+5U165d1bdvX5UqVUoxMTFauHChZs+erUuXLhV2mdq6davGjx+vQYMGyc/Pr7DLgc0QiIBC5unpWdgl3LDU1FSVLFmysMvItVOnTsnb2/uqYUiSgoKC1L9/f6e2N954QwEBAdnabzWDBg3Srl279MUXX6hXr15O2yZMmHDLBz7AFbhkBhSyK+8hSk9P1/jx41W9enV5eXmpbNmyatGihaKioiT9+cttxowZkpTjZZzU1FSNHDlSFStWlKenp2rUqKG33npLxhinz71w4YKGDRumgIAAlS5dWg888IB+//13ORwOpzNP48aNk8Ph0P79+9W3b1+VKVNGLVq0kCTt3r1bgwYNUtWqVeXl5aXg4GA9+uijOn36tNNnZY1x6NAh9e/fX76+vipXrpz+/e9/yxij48ePq1u3bvLx8VFwcLCmTJmSq7m7fPmyJkyYoDvuuEOenp6qXLmy/vWvfyktLc3q43A4NHfuXKWmplpzNW/evFyNn5Off/5ZDz30kPz9/VWiRAk1a9ZMy5cvv+5+aWlp6tq1q3x9fbV161ZJUmZmpqZPn646derIy8tLQUFBeuKJJ3T27FmnfStXrqyuXbvq22+/VZMmTeTl5aWqVatqwYIF1/3c7du3a/ny5Ro8eHC2MCT9Gcjfeustp7Z169apZcuWKlmypPz8/NStWzcdOHDAqc+gQYNUuXLlbONl/bf+K4fDocjISH311VeqW7euPD09VadOHa1atcppv1GjRkmSqlSpYv23+uWXXyRJUVFRatGihfz8/FSqVCnVqFFD//rXv657/EBucYYIyAdJSUlKSEjI1p6enn7dfceNG6eJEyfqscceU5MmTZScnKwdO3boxx9/1H333acnnnhCJ06cUFRUVLZLPMYYPfDAA1q/fr0GDx6shg0b6ptvvtGoUaP0+++/a9q0aVbfQYMG6fPPP9fDDz+sZs2aaePGjerSpctV63rooYdUvXp1vf7661a4ioqK0s8//6xHHnlEwcHB2rdvn2bPnq19+/bpu+++y/aL8Z///Kdq1aqlN954Q8uXL9err74qf39/ffDBB7r33nv15ptv6pNPPtFzzz2nxo0bq1WrVtecq8cee0zz58/Xgw8+qJEjR2r79u2aOHGiDhw4oCVLlkiS/vOf/2j27Nn6/vvv9eGHH0qS7rnnnuv+d8jJyZMndc899+j8+fMaNmyYypYtq/nz5+uBBx7QF198oR49euS434ULF9StWzft2LFDa9asUePGjSVJTzzxhObNm6dHHnlEw4YN07Fjx/Tee+9p165d2rJli9zd3a0xjhw5ogcffFCDBw/WwIEDNWfOHA0aNEiNGjVSnTp1rlrz0qVLJUkPP/xwro5xzZo16tSpk6pWrapx48bpwoULevfdd9W8eXP9+OOPOYag3Pj222/15Zdf6umnn1bp0qX1zjvvqFevXoqNjVXZsmXVs2dPHTp0SJ999pmmTZumgIAASVK5cuW0b98+de3aVfXr19crr7wiT09PHTlyRFu2bMlTLUCODACXmTt3rpF0zaVOnTpO+4SGhpqBAwda6w0aNDBdunS55udERESYnP76fvXVV0aSefXVV53aH3zwQeNwOMyRI0eMMcbs3LnTSDLDhw936jdo0CAjyYwdO9ZqGzt2rJFk+vTpk+3zzp8/n63ts88+M5LMpk2bso0xZMgQq+3y5cumQoUKxuFwmDfeeMNqP3v2rPH29naak5xER0cbSeaxxx5zan/uueeMJLNu3TqrbeDAgaZkyZLXHC8nderUMa1bt7bWhw8fbiSZzZs3W23nzp0zVapUMZUrVzYZGRnGGGPWr19vJJnFixebc+fOmdatW5uAgACza9cua7/NmzcbSeaTTz5x+sxVq1Zlaw8NDc02p6dOnTKenp5m5MiR1zyGHj16GEnm7NmzuTrmhg0bmsDAQHP69Gmr7aeffjJubm5mwIABVtvAgQNNaGhotv2z/lv/lSTj4eFh/fnLGlOSeffdd622yZMnG0nm2LFjTvtPmzbNSDJ//PFHro4ByAsumQH5YMaMGYqKisq21K9f/7r7+vn5ad++fTp8+PANf+6KFStUrFgxDRs2zKl95MiRMsZo5cqVkmRdqnj66aed+g0dOvSqYz/55JPZ2ry9va1/v3jxohISEtSsWTNJ0o8//pit/2OPPWb9e7FixXT33XfLGKPBgwdb7X5+fqpRo4Z+/vnnq9Yi/XmskjRixAin9pEjR0pSri5j3agVK1aoSZMm1iVDSSpVqpSGDBmiX375Rfv373fqn5SUpA4dOujgwYPasGGD0+P+ixcvlq+vr+677z4lJCRYS6NGjVSqVCmtX7/eaazatWurZcuW1nq5cuVyNU/JycmSpNKlS1/3+OLi4hQdHa1BgwbJ39/faq9fv77uu+8+a87zon379rrjjjucxvTx8blu/ZKsG6y//vprZWZm5rkG4FoIREA+aNKkidq3b59tKVOmzHX3feWVV5SYmKg777xT9erV06hRo7R79+5cfe6vv/6qkJCQbL/8atWqZW3P+qebm5uqVKni1K9atWpXHfvKvpJ05swZPfPMMwoKCpK3t7fKlStn9UtKSsrWv1KlSk7rWY9/Z10e+Wv7lffRXCnrGK6sOTg4WH5+ftaxutKvv/6qGjVqZGu/cn6zDB8+XD/88IPWrFmT7bLW4cOHlZSUpMDAQJUrV85pSUlJ0alTp5z6Xzl3klSmTJnrzpOPj48k6dy5c7k6PklXPcaEhASlpqZed5yc5LV+6c9Lrc2bN9djjz2moKAg9e7dW59//jnhCC7FPURAEdOqVSsdPXpUX3/9tVavXq0PP/xQ06ZN06xZs5zOsBS0v54NyvKPf/xDW7du1ahRo9SwYUOVKlVKmZmZ6tixY46/rIoVK5arNknZbgK/mqL8vUDdunXTwoUL9cYbb2jBggVOX6+QmZmpwMBAffLJJznuW65cOaf1vM5TzZo1JUl79uxxOsN0s6427xkZGTm238x/Z29vb23atEnr16/X8uXLtWrVKi1atEj33nuvVq9efdWxgRvBGSKgCPL399cjjzyizz77TMePH1f9+vWdnvy62i+j0NBQnThxItvZgIMHD1rbs/6ZmZmpY8eOOfU7cuRIrms8e/as1q5dqxdffFHjx49Xjx49dN9996lq1aq5HuNmZB3DlZcWT548qcTEROtYXf2ZMTEx2dqvnN8s3bt315w5c/Tpp58qIiLCadsdd9yh06dPq3nz5jmeTWzQoIFLar7//vslSR9//PF1+2bVf7VjDAgIsL5uoUyZMjl+geLNnJm7Vrh1c3NTu3btNHXqVO3fv1+vvfaa1q1bl+3SIpBXBCKgiLnykfVSpUqpWrVqTo+SZ/1SuvIXUufOnZWRkaH33nvPqX3atGlyOBzq1KmTJCk8PFyS9P777zv1e/fdd3NdZ9b/lV/5f/jTp0/P9Rg3o3Pnzjl+3tSpUyXpmk/M3cxnfv/999q2bZvVlpqaqtmzZ6ty5cqqXbt2tn0GDBigd955R7NmzdILL7xgtf/jH/9QRkaGJkyYkG2fy5cvu+zbmsPCwtSxY0d9+OGH+uqrr7Jtv3Tpkp577jlJUvny5dWwYUPNnz/f6fP37t2r1atXW3Mu/RnokpKSnC7nxsXFWU/35cXV/lyfOXMmW9+s+7H++vcCuBlcMgOKmNq1a6tNmzZq1KiR/P39tWPHDn3xxReKjIy0+jRq1EiSNGzYMIWHh6tYsWLq3bu37r//frVt21YvvfSSfvnlFzVo0ECrV6/W119/reHDh1s3tTZq1Ei9evXS9OnTdfr0aeux+6xXOOTmMpSPj49atWqlSZMmKT09XX/729+0evXqbGed8kuDBg00cOBAzZ49W4mJiWrdurW+//57zZ8/X927d1fbtm1d/pkvvviiPvvsM3Xq1EnDhg2Tv7+/5s+fr2PHjum///3vVb9xPDIyUsnJyXrppZfk6+urf/3rX2rdurWeeOIJTZw4UdHR0erQoYPc3d11+PBhLV68WG+//bYefPBBl9S9YMECdejQQT179tT999+vdu3aqWTJkjp8+LAWLlyouLg467uIJk+erE6dOiksLEyDBw+2Hrv39fV1OkvZu3dvvfDCC+rRo4eGDRum8+fPa+bMmbrzzjtzvKE+N7L+XL/00kvq3bu33N3ddf/99+uVV17Rpk2b1KVLF4WGhurUqVN6//33VaFCBacb3IGbUpiPuAG3m6zH7n/44Ycct7du3fq6j92/+uqrpkmTJsbPz894e3ubmjVrmtdee81cunTJ6nP58mUzdOhQU65cOeNwOJwecz537px59tlnTUhIiHF3dzfVq1c3kydPNpmZmU6fm5qaaiIiIoy/v78pVaqU6d69u4mJiTGSnB6Dz3qMOqdHnn/77TfTo0cP4+fnZ3x9fc1DDz1kTpw4cdVH968c42qPw+c0TzlJT08348ePN1WqVDHu7u6mYsWKZvTo0ebixYu5+pzrufKxe2OMOXr0qHnwwQeNn5+f8fLyMk2aNDHLli1z6vPXx+7/6vnnnzeSzHvvvWe1zZ492zRq1Mh4e3ub0qVLm3r16pnnn3/enDhxwuoTGhqa41cxtG7dOlt9V3P+/Hnz1ltvmcaNG5tSpUoZDw8PU716dTN06FCnx+GNMWbNmjWmefPmxtvb2/j4+Jj777/f7N+/P9uYq1evNnXr1jUeHh6mRo0a5uOPP77qY/cRERHZ9r/yz74xxkyYMMH87W9/M25ubtYj+GvXrjXdunUzISEhxsPDw4SEhJg+ffqYQ4cO5erYgdxwGJPLOxcB3Paio6P197//XR9//LH69etX2OUAQIHhHiLApi5cuJCtbfr06XJzc7vuN0QDwO2Ge4gAm5o0aZJ27typtm3bqnjx4lq5cqVWrlypIUOGqGLFioVdHgAUKC6ZATYVFRWl8ePHa//+/UpJSVGlSpX08MMP66WXXlLx4vy/EgB7IRABAADb4x4iAABgewQiAABge9wokAuZmZk6ceKESpcuXaTfmwQAAP6fMUbnzp1TSEjIVb84NQuBKBdOnDjBUzcAANyijh8/rgoVKlyzD4EoF0qXLi3pzwn18fEp5GoAAEBuJCcnq2LFitbv8WshEOVC1mUyHx8fAhEAALeY3Nzuwk3VAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9go1EE2cOFGNGzdW6dKlFRgYqO7duysmJsapT5s2beRwOJyWJ5980qlPbGysunTpohIlSigwMFCjRo3S5cuXnfps2LBBd911lzw9PVWtWjXNmzcvvw8PAADcIgo1EG3cuFERERH67rvvFBUVpfT0dHXo0EGpqalO/R5//HHFxcVZy6RJk6xtGRkZ6tKliy5duqStW7dq/vz5mjdvnl5++WWrz7Fjx9SlSxe1bdtW0dHRGj58uB577DF98803BXasAACg6CpSb7v/448/FBgYqI0bN6pVq1aS/jxD1LBhQ02fPj3HfVauXKmuXbvqxIkTCgoKkiTNmjVLL7zwgv744w95eHjohRde0PLly7V3715rv969eysxMVGrVq26bl3Jycny9fVVUlIS30MEAMAt4kZ+fxepe4iSkpIkSf7+/k7tn3zyiQICAlS3bl2NHj1a58+ft7Zt27ZN9erVs8KQJIWHhys5OVn79u2z+rRv395pzPDwcG3bti3HOtLS0pScnOy0AACA21eR+abqzMxMDR8+XM2bN1fdunWt9r59+yo0NFQhISHavXu3XnjhBcXExOjLL7+UJMXHxzuFIUnWenx8/DX7JCcn68KFC/L29nbaNnHiRI0fP97lxwgAAIqmIhOIIiIitHfvXn377bdO7UOGDLH+vV69eipfvrzatWuno0eP6o477siXWkaPHq0RI0ZY61nvQgEAALenInHJLDIyUsuWLdP69euv+zbapk2bSpKOHDkiSQoODtbJkyed+mStBwcHX7OPj49PtrNDkuTp6Wm9t4z3lwEAcPsr1EBkjFFkZKSWLFmidevWqUqVKtfdJzo6WpJUvnx5SVJYWJj27NmjU6dOWX2ioqLk4+Oj2rVrW33Wrl3rNE5UVJTCwsJcdCQAAOBWVqiBKCIiQh9//LE+/fRTlS5dWvHx8YqPj9eFCxckSUePHtWECRO0c+dO/fLLL1q6dKkGDBigVq1aqX79+pKkDh06qHbt2nr44Yf1008/6ZtvvtGYMWMUEREhT09PSdKTTz6pn3/+Wc8//7wOHjyo999/X59//rmeffbZQjt2AABQdBTqY/cOhyPH9rlz52rQoEE6fvy4+vfvr7179yo1NVUVK1ZUjx49NGbMGKfLWL/++queeuopbdiwQSVLltTAgQP1xhtvqHjx/79FasOGDXr22We1f/9+VahQQf/+9781aNCgXNXJY/fZxcbGKiEhIV/GDggIUKVKlfJlbACAfdzI7+8i9T1ERRWByFlsbKxq1KylixfOX79zHnh5l1DMwQOEIgDATbmR399F5ikz3DoSEhJ08cJ5le06Uu5lXfv0Xfrp4zq9bIoSEhIIRACAAkMgQp65l60oz+BqhV0GAAA3rUg8dg8AAFCYCEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2ihd2AUBODhw44PIxAwICVKlSJZePCwC49RGIUKRkpJyVHA7179/f5WN7eZdQzMEDhCIAQDYEIhQpmWkpkjEq23Wk3MtWdNm46aeP6/SyKUpISCAQAQCyIRChSHIvW1GewdUKuwwAgE1wUzUAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALC9Qg1EEydOVOPGjVW6dGkFBgaqe/fuiomJcepz8eJFRUREqGzZsipVqpR69eqlkydPOvWJjY1Vly5dVKJECQUGBmrUqFG6fPmyU58NGzborrvukqenp6pVq6Z58+bl9+EBAIBbRKEGoo0bNyoiIkLfffedoqKilJ6erg4dOig1NdXq8+yzz+p///d/tXjxYm3cuFEnTpxQz549re0ZGRnq0qWLLl26pK1bt2r+/PmaN2+eXn75ZavPsWPH1KVLF7Vt21bR0dEaPny4HnvsMX3zzTcFerwAAKBoKtQvZly1apXT+rx58xQYGKidO3eqVatWSkpK0kcffaRPP/1U9957ryRp7ty5qlWrlr777js1a9ZMq1ev1v79+7VmzRoFBQWpYcOGmjBhgl544QWNGzdOHh4emjVrlqpUqaIpU6ZIkmrVqqVvv/1W06ZNU3h4eIEfNwAAKFqK1D1ESUlJkiR/f39J0s6dO5Wenq727dtbfWrWrKlKlSpp27ZtkqRt27apXr16CgoKsvqEh4crOTlZ+/bts/r8dYysPlljXCktLU3JyclOCwAAuH0VmUCUmZmp4cOHq3nz5qpbt64kKT4+Xh4eHvLz83PqGxQUpPj4eKvPX8NQ1vasbdfqk5ycrAsXLmSrZeLEifL19bWWihVd904tAABQ9BSZQBQREaG9e/dq4cKFhV2KRo8eraSkJGs5fvx4YZcEAADyUZF4uWtkZKSWLVumTZs2qUKFClZ7cHCwLl26pMTERKezRCdPnlRwcLDV5/vvv3caL+sptL/2ufLJtJMnT8rHx0fe3t7Z6vH09JSnp6dLjg0AABR9hXqGyBijyMhILVmyROvWrVOVKlWctjdq1Eju7u5au3at1RYTE6PY2FiFhYVJksLCwrRnzx6dOnXK6hMVFSUfHx/Vrl3b6vPXMbL6ZI0BAADsrVDPEEVEROjTTz/V119/rdKlS1v3/Pj6+srb21u+vr4aPHiwRowYIX9/f/n4+Gjo0KEKCwtTs2bNJEkdOnRQ7dq19fDDD2vSpEmKj4/XmDFjFBERYZ3lefLJJ/Xee+/p+eef16OPPqp169bp888/1/Llywvt2AEAQNFRqGeIZs6cqaSkJLVp00bly5e3lkWLFll9pk2bpq5du6pXr15q1aqVgoOD9eWXX1rbixUrpmXLlqlYsWIKCwtT//79NWDAAL3yyitWnypVqmj58uWKiopSgwYNNGXKFH344Yc8cg8AACQV8hkiY8x1+3h5eWnGjBmaMWPGVfuEhoZqxYoV1xynTZs22rVr1w3XCAAAbn9F5ikzAACAwkIgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtleogWjTpk26//77FRISIofDoa+++spp+6BBg+RwOJyWjh07OvU5c+aM+vXrJx8fH/n5+Wnw4MFKSUlx6rN79261bNlSXl5eqlixoiZNmpTfhwYAAG4heQpEP//8s0s+PDU1VQ0aNNCMGTOu2qdjx46Ki4uzls8++8xpe79+/bRv3z5FRUVp2bJl2rRpk4YMGWJtT05OVocOHRQaGqqdO3dq8uTJGjdunGbPnu2SYwAAALe+4nnZqVq1amrdurUGDx6sBx98UF5eXnn68E6dOqlTp07X7OPp6ang4OActx04cECrVq3SDz/8oLvvvluS9O6776pz58566623FBISok8++USXLl3SnDlz5OHhoTp16ig6OlpTp051Ck4AAMC+8nSG6Mcff1T9+vU1YsQIBQcH64knntD333/v6tokSRs2bFBgYKBq1Kihp556SqdPn7a2bdu2TX5+flYYkqT27dvLzc1N27dvt/q0atVKHh4eVp/w8HDFxMTo7NmzOX5mWlqakpOTnRYAAHD7ylMgatiwod5++22dOHFCc+bMUVxcnFq0aKG6detq6tSp+uOPP1xSXMeOHbVgwQKtXbtWb775pjZu3KhOnTopIyNDkhQfH6/AwECnfYoXLy5/f3/Fx8dbfYKCgpz6ZK1n9bnSxIkT5evray0VK1Z0yfEAAICi6aZuqi5evLh69uypxYsX680339SRI0f03HPPqWLFihowYIDi4uJuqrjevXvrgQceUL169dS9e3ctW7ZMP/zwgzZs2HBT417P6NGjlZSUZC3Hjx/P188DAACF66YC0Y4dO/T000+rfPnymjp1qp577jkdPXpUUVFROnHihLp16+aqOiVJVatWVUBAgI4cOSJJCg4O1qlTp5z6XL58WWfOnLHuOwoODtbJkyed+mStX+3eJE9PT/n4+DgtAADg9pWnQDR16lTVq1dP99xzj06cOKEFCxbo119/1auvvqoqVaqoZcuWmjdvnn788UeXFvvbb7/p9OnTKl++vCQpLCxMiYmJ2rlzp9Vn3bp1yszMVNOmTa0+mzZtUnp6utUnKipKNWrUUJkyZVxaHwAAuDXlKRDNnDlTffv21a+//qqvvvpKXbt2lZub81CBgYH66KOPrjlOSkqKoqOjFR0dLUk6duyYoqOjFRsbq5SUFI0aNUrfffedfvnlF61du1bdunVTtWrVFB4eLkmqVauWOnbsqMcff1zff/+9tmzZosjISPXu3VshISGSpL59+8rDw0ODBw/Wvn37tGjRIr399tsaMWJEXg4dAADchvL02P3hw4ev28fDw0MDBw68Zp8dO3aobdu21npWSBk4cKBmzpyp3bt3a/78+UpMTFRISIg6dOigCRMmyNPT09rnk08+UWRkpNq1ayc3Nzf16tVL77zzjrXd19dXq1evVkREhBo1aqSAgAC9/PLLPHIPAAAseQpEc+fOValSpfTQQw85tS9evFjnz5+/bhDK0qZNGxljrrr9m2++ue4Y/v7++vTTT6/Zp379+tq8eXOuagIAAPaTp0tmEydOVEBAQLb2wMBAvf766zddFAAAQEHKUyCKjY1VlSpVsrWHhoYqNjb2posCAAAoSHkKRIGBgdq9e3e29p9++klly5a96aIAAAAKUp4CUZ8+fTRs2DCtX79eGRkZysjI0Lp16/TMM8+od+/erq4RAAAgX+XppuoJEybol19+Ubt27VS8+J9DZGZmasCAAdxDBAAAbjl5CkQeHh5atGiRJkyYoJ9++kne3t6qV6+eQkNDXV0fAABAvstTIMpy55136s4773RVLQAAAIUiT4EoIyND8+bN09q1a3Xq1CllZmY6bV+3bp1LigMAACgIeQpEzzzzjObNm6cuXbqobt26cjgcrq4LAACgwOQpEC1cuFCff/65Onfu7Op64EKxsbFKSEhw+bgHDhxw+ZgAABSmPN9UXa1aNVfXAheKjY1VjZq1dPHC+cIuBQCAIi9PgWjkyJF6++239d5773G5rIhKSEjQxQvnVbbrSLmXrejSsS/8vENJmz926ZgAABSmPAWib7/9VuvXr9fKlStVp04dubu7O23/8ssvXVIcbp572YryDHbt2bz008ddOh4AAIUtT4HIz89PPXr0cHUtAAAAhSJPgWju3LmurgMAAKDQ5OldZpJ0+fJlrVmzRh988IHOnTsnSTpx4oRSUlJcVhwAAEBByNMZol9//VUdO3ZUbGys0tLSdN9996l06dJ68803lZaWplmzZrm6TgAAgHyTpzNEzzzzjO6++26dPXtW3t7eVnuPHj20du1alxUHAABQEPJ0hmjz5s3aunWrPDw8nNorV66s33//3SWFAQAAFJQ8nSHKzMxURkZGtvbffvtNpUuXvumiAAAAClKeAlGHDh00ffp0a93hcCglJUVjx47ldR4AAOCWk6dLZlOmTFF4eLhq166tixcvqm/fvjp8+LACAgL02WefubpGAACAfJWnQFShQgX99NNPWrhwoXbv3q2UlBQNHjxY/fr1c7rJGgAA4FaQp0AkScWLF1f//v1dWQsAAEChyFMgWrBgwTW3DxgwIE/FAAAAFIY8BaJnnnnGaT09PV3nz5+Xh4eHSpQoQSACAAC3lDw9ZXb27FmnJSUlRTExMWrRogU3VQMAgFtOnt9ldqXq1avrjTfeyHb2CAAAoKhzWSCS/rzR+sSJE64cEgAAIN/l6R6ipUuXOq0bYxQXF6f33ntPzZs3d0lhAAAABSVPgah79+5O6w6HQ+XKldO9996rKVOmuKIuAACAApOnQJSZmenqOgAAAAqNS+8hAgAAuBXl6QzRiBEjct136tSpefkIIF8cOHAgX8YNCAhQpUqV8mVsAED+y1Mg2rVrl3bt2qX09HTVqFFDknTo0CEVK1ZMd911l9XP4XC4pkrgJmWknJUcjnx73YyXdwnFHDxAKAKAW1SeAtH999+v0qVLa/78+SpTpoykP7+s8ZFHHlHLli01cuRIlxYJ3KzMtBTJGJXtOlLuZSu6dOz008d1etkUJSQkEIgA4BaVp0A0ZcoUrV692gpDklSmTBm9+uqr6tChA4EIRZZ72YryDK5W2GUAAIqYPN1UnZycrD/++CNb+x9//KFz587ddFEAAAAFKU+BqEePHnrkkUf05Zdf6rffftNvv/2m//73vxo8eLB69uzp6hoBAADyVZ4umc2aNUvPPfec+vbtq/T09D8HKl5cgwcP1uTJk11aIAAAQH7LUyAqUaKE3n//fU2ePFlHjx6VJN1xxx0qWbKkS4sDAAAoCDf1xYxxcXGKi4tT9erVVbJkSRljXFUXAABAgclTIDp9+rTatWunO++8U507d1ZcXJwkafDgwTxhBgAAbjl5CkTPPvus3N3dFRsbqxIlSljt//znP7Vq1SqXFQcAAFAQ8nQP0erVq/XNN9+oQoUKTu3Vq1fXr7/+6pLCAAAACkqezhClpqY6nRnKcubMGXl6et50UQAAAAUpT4GoZcuWWrBggbXucDiUmZmpSZMmqW3bti4rDgAAoCDk6ZLZpEmT1K5dO+3YsUOXLl3S888/r3379unMmTPasmWLq2sEAADIV3k6Q1S3bl0dOnRILVq0ULdu3ZSamqqePXtq165duuOOO1xdIwAAQL664TNE6enp6tixo2bNmqWXXnopP2oCAAAoUDd8hsjd3V27d+/Oj1oAAAAKRZ4umfXv318fffSRq2sBAAAoFHm6qfry5cuaM2eO1qxZo0aNGmV7h9nUqVNdUhwAAEBBuKFA9PPPP6ty5crau3ev7rrrLknSoUOHnPo4HA7XVQcAAFAAbigQVa9eXXFxcVq/fr2kP1/V8c477ygoKChfigMAACgIN3QP0ZVvs1+5cqVSU1NdWhAAAEBBy9NN1VmuDEgAAAC3ohsKRA6HI9s9QtwzBAAAbnU3dA+RMUaDBg2yXuB68eJFPfnkk9meMvvyyy9dVyEAAEA+u6FANHDgQKf1/v37u7QYAACAwnBDgWju3Lku/fBNmzZp8uTJ2rlzp+Li4rRkyRJ1797d2m6M0dixY/U///M/SkxMVPPmzTVz5kxVr17d6nPmzBkNHTpU//u//ys3Nzf16tVLb7/9tkqVKmX12b17tyIiIvTDDz+oXLlyGjp0qJ5//nmXHgsAALh13dRN1TcrNTVVDRo00IwZM3LcPmnSJL3zzjuaNWuWtm/frpIlSyo8PFwXL160+vTr10/79u1TVFSUli1bpk2bNmnIkCHW9uTkZHXo0EGhoaHauXOnJk+erHHjxmn27Nn5fnwAAODWkKdvqnaVTp06qVOnTjluM8Zo+vTpGjNmjLp16yZJWrBggYKCgvTVV1+pd+/eOnDggFatWqUffvhBd999tyTp3XffVefOnfXWW28pJCREn3zyiS5duqQ5c+bIw8NDderUUXR0tKZOneoUnAAAgH0V6hmiazl27Jji4+PVvn17q83X11dNmzbVtm3bJEnbtm2Tn5+fFYYkqX379nJzc9P27dutPq1atZKHh4fVJzw8XDExMTp79mwBHQ0AACjKCvUM0bXEx8dLUrZvwQ4KCrK2xcfHKzAw0Gl78eLF5e/v79SnSpUq2cbI2lamTJlsn52Wlqa0tDRrPTk5+SaPBgAAFGVF9gxRYZo4caJ8fX2tpWLFioVdEgAAyEdFNhAFBwdLkk6ePOnUfvLkSWtbcHCwTp065bT98uXLOnPmjFOfnMb462dcafTo0UpKSrKW48eP3/wBAQCAIqvIBqIqVaooODhYa9eutdqSk5O1fft2hYWFSZLCwsKUmJionTt3Wn3WrVunzMxMNW3a1OqzadMmpaenW32ioqJUo0aNHC+XSZKnp6d8fHycFgAAcPsq1ECUkpKi6OhoRUdHS/rzRuro6GjFxsbK4XBo+PDhevXVV7V06VLt2bNHAwYMUEhIiPVdRbVq1VLHjh31+OOP6/vvv9eWLVsUGRmp3r17KyQkRJLUt29feXh4aPDgwdq3b58WLVqkt99+WyNGjCikowYAAEVNod5UvWPHDrVt29ZazwopAwcO1Lx58/T8888rNTVVQ4YMUWJiolq0aKFVq1bJy8vL2ueTTz5RZGSk2rVrZ30x4zvvvGNt9/X11erVqxUREaFGjRopICBAL7/8Mo/cAwAAS6EGojZt2sgYc9XtDodDr7zyil555ZWr9vH399enn356zc+pX7++Nm/enOc6AQDA7a3I3kMEAABQUAhEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ooXdgHA7eLAgQMuHzMgIECVKlVy+bgAAGcEIuAmZaSclRwO9e/f3+Vje3mXUMzBA4QiAMhnBCLgJmWmpUjGqGzXkXIvW9Fl46afPq7Ty6YoISGBQAQA+YxABLiIe9mK8gyuVthlAADygJuqAQCA7RGIAACA7RGIAACA7RGIAACA7RGIAACA7RGIAACA7RGIAACA7RGIAACA7RGIAACA7RGIAACA7RGIAACA7RGIAACA7RGIAACA7RGIAACA7RGIAACA7RGIAACA7RXpQDRu3Dg5HA6npWbNmtb2ixcvKiIiQmXLllWpUqXUq1cvnTx50mmM2NhYdenSRSVKlFBgYKBGjRqly5cvF/ShAACAIqx4YRdwPXXq1NGaNWus9eLF/7/kZ599VsuXL9fixYvl6+uryMhI9ezZU1u2bJEkZWRkqEuXLgoODtbWrVsVFxenAQMGyN3dXa+//nqBHwsAACiainwgKl68uIKDg7O1JyUl6aOPPtKnn36qe++9V5I0d+5c1apVS999952aNWum1atXa//+/VqzZo2CgoLUsGFDTZgwQS+88ILGjRsnDw+Pgj4cAABQBBXpS2aSdPjwYYWEhKhq1arq16+fYmNjJUk7d+5Uenq62rdvb/WtWbOmKlWqpG3btkmStm3bpnr16ikoKMjqEx4eruTkZO3bt++qn5mWlqbk5GSnBQAA3L6KdCBq2rSp5s2bp1WrVmnmzJk6duyYWrZsqXPnzik+Pl4eHh7y8/Nz2icoKEjx8fGSpPj4eKcwlLU9a9vVTJw4Ub6+vtZSsWJF1x4YAAAoUor0JbNOnTpZ/16/fn01bdpUoaGh+vzzz+Xt7Z1vnzt69GiNGDHCWk9OTiYUAQBwGyvSZ4iu5OfnpzvvvFNHjhxRcHCwLl26pMTERKc+J0+etO45Cg4OzvbUWdZ6TvclZfH09JSPj4/TAgAAbl+3VCBKSUnR0aNHVb58eTVq1Eju7u5au3attT0mJkaxsbEKCwuTJIWFhWnPnj06deqU1ScqKko+Pj6qXbt2gdcPAACKpiJ9yey5557T/fffr9DQUJ04cUJjx45VsWLF1KdPH/n6+mrw4MEaMWKE/P395ePjo6FDhyosLEzNmjWTJHXo0EG1a9fWww8/rEmTJik+Pl5jxoxRRESEPD09C/noAABAUVGkA9Fvv/2mPn366PTp0ypXrpxatGih7777TuXKlZMkTZs2TW5uburVq5fS0tIUHh6u999/39q/WLFiWrZsmZ566imFhYWpZMmSGjhwoF555ZXCOiQAAFAEFelAtHDhwmtu9/Ly0owZMzRjxoyr9gkNDdWKFStcXRoAALiN3FL3EAEAAOQHAhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALC94oVdAIBrO3DgQL6MGxAQoEqVKuXL2ABwqyEQAUVURspZyeFQ//7982V8L+8Sijl4gFAEACIQAUVWZlqKZIzKdh0p97IVXTp2+unjOr1sihISEghEACACEVDkuZetKM/gaoVdBgDc1ripGgAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2F7xwi4AUmxsrBISElw65oEDB1w6HgAAtzMCUSGLjY1VjZq1dPHC+cIuBQAA2yIQFbKEhARdvHBeZbuOlHvZii4b98LPO5S0+WOXjYfbU36cSQwICFClSpVcPi4A5CcCURHhXraiPIOruWy89NPHXTYWbj8ZKWclh0P9+/d3+dhe3iUUc/AAoQjALYVABNhQZlqKZIzLz0ymnz6u08umKCEhgUAE4JZCIAJszNVnJgHgVsVj9wAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPZ47B6Ay+XXu/T4FmwA+YVABMBl8vMbsCW+BRtA/rFVIJoxY4YmT56s+Ph4NWjQQO+++66aNGlS2GUBt438+gZsiW/BBpC/bBOIFi1apBEjRmjWrFlq2rSppk+frvDwcMXExCgwMLCwywNuK/n5Ddi8kBZAfrBNIJo6daoef/xxPfLII5KkWbNmafny5ZozZ45efPHFQq4OwPXwQloA+ckWgejSpUvauXOnRo8ebbW5ubmpffv22rZtWyFWBiC3eCEtriU2NlYJCQn5MjZnEO3BFoEoISFBGRkZCgoKcmoPCgrSwYMHs/VPS0tTWlqatZ6UlCRJSk5OdnltKSkpf35m/BFlXrrosnHTTx/Pl3Fv1bGpuWDGLoiaM9PTXDp2Zvqff9d37txp/X10JTc3N2VmZt4y496KY588eVL9Hx6gS2mu/TOXxcPTSx//Z0G23yE361ab5/wcV5KCg4MVHBzs0jGzfm8bY67f2djA77//biSZrVu3OrWPGjXKNGnSJFv/sWPHGkksLCwsLCwst8Fy/Pjx62YFW5whCggIULFixXTy5Emn9pMnT+aYRkePHq0RI0ZY65mZmTpz5ozKli0rh8OR7/Xmh+TkZFWsWFHHjx+Xj49PYZdTpDFXN4b5yj3mKveYq9xjrq7OGKNz584pJCTkun1tEYg8PDzUqFEjrV27Vt27d5f0Z8hZu3atIiMjs/X39PSUp6enU5ufn18BVJr/fHx8+AuTS8zVjWG+co+5yj3mKveYq5z5+vrmqp8tApEkjRgxQgMHDtTdd9+tJk2aaPr06UpNTbWeOgMAAPZlm0D0z3/+U3/88YdefvllxcfHq2HDhlq1apXLb5IDAAC3HtsEIkmKjIzM8RKZHXh6emrs2LHZLgUiO+bqxjBfucdc5R5zlXvMlWs4jMnNs2gAAAC3L7fCLgAAAKCwEYgAAIDtEYgAAIDtEYgAAIDtEYhuYRMnTlTjxo1VunRpBQYGqnv37oqJiXHqc/HiRUVERKhs2bIqVaqUevXqle0bu2NjY9WlSxeVKFFCgYGBGjVqlC5fvlyQh5LvrjdXZ86c0dChQ1WjRg15e3urUqVKGjZsmPUeuyzMlTNjjDp16iSHw6GvvvrKaZsd5krK/Xxt27ZN9957r0qWLCkfHx+1atVKFy5csLafOXNG/fr1k4+Pj/z8/DR48OB8ebdaYcrNXMXHx+vhhx9WcHCwSpYsqbvuukv//e9/nfrYYa5mzpyp+vXrW1+2GBYWppUrV1rb+dmeD1zysjAUivDwcDN37lyzd+9eEx0dbTp37mwqVapkUlJSrD5PPvmkqVixolm7dq3ZsWOHadasmbnnnnus7ZcvXzZ169Y17du3N7t27TIrVqwwAQEBZvTo0YVxSPnmenO1Z88e07NnT7N06VJz5MgRs3btWlO9enXTq1cvawzmKiVb36lTp5pOnToZSWbJkiVWu13mypjczdfWrVuNj4+PmThxotm7d685ePCgWbRokbl48aLVp2PHjqZBgwbmu+++M5s3bzbVqlUzffr0KYxDyje5mav77rvPNG7c2Gzfvt0cPXrUTJgwwbi5uZkff/zR6mOHuVq6dKlZvny5OXTokImJiTH/+te/jLu7u9m7d68xhp/t+YFAdBs5deqUkWQ2btxojDEmMTHRuLu7m8WLF1t9Dhw4YCSZbdu2GWOMWbFihXFzczPx8fFWn5kzZxofHx+TlpZWsAdQgK6cq5x8/vnnxsPDw6SnpxtjmKsr52rXrl3mb3/7m4mLi8sWiOw6V8bkPF9NmzY1Y8aMueo++/fvN5LMDz/8YLWtXLnSOBwO8/vvv+drvYUpp7kqWbKkWbBggVM/f39/8z//8z/GGPvOlTHGlClTxnz44Yf8bM8nXDK7jWRd3vH395ck7dy5U+np6Wrfvr3Vp2bNmqpUqZK2bdsm6c/T+PXq1XP6xu7w8HAlJydr3759BVh9wbpyrq7Wx8fHR8WL//n9pczV/8/V+fPn1bdvX82YMSPHFyTbda6k7PN16tQpbd++XYGBgbrnnnsUFBSk1q1b69tvv7X22bZtm/z8/HT33Xdbbe3bt5ebm5u2b99esAdQgHL6s3XPPfdo0aJFOnPmjDIzM7Vw4UJdvHhRbdq0kWTPucrIyNDChQuVmpqqsLAwfrbnEwLRbSIzM1PDhw9X8+bNVbduXUl/Xov38PDI9mLaoKAgxcfHW32ufH1J1npWn9tNTnN1pYSEBE2YMEFDhgyx2pir/5+rZ599Vvfcc4+6deuW4352nCsp5/n6+eefJUnjxo3T448/rlWrVumuu+5Su3btdPjwYUl/zklgYKDTWMWLF5e/v/9tO19X+7P1+eefKz09XWXLlpWnp6eeeOIJLVmyRNWqVZNkr7nas2ePSpUqJU9PTz355JNasmSJateuzc/2fGKrV3fcziIiIrR3716n/+tEzq43V8nJyerSpYtq166tcePGFWxxRUxOc7V06VKtW7dOu3btKsTKiqac5iszM1OS9MQTT1gvk/773/+utWvXas6cOZo4cWKh1FrYrvb38N///rcSExO1Zs0aBQQE6KuvvtI//vEPbd68WfXq1SukagtHjRo1FB0draSkJH3xxRcaOHCgNm7cWNhl3bY4Q3QbiIyM1LJly7R+/XpVqFDBag8ODtalS5eUmJjo1P/kyZPWZY7g4OBsTyZkred0KeRWd7W5ynLu3Dl17NhRpUuX1pIlS+Tu7m5tY67+tG7dOh09elR+fn4qXry4dUmxV69e1mUNu82VdPX5Kl++vCSpdu3aTv1r1aql2NhYSX/OyalTp5y2X758WWfOnLkt5+tqc3X06FG99957mjNnjtq1a6cGDRpo7NixuvvuuzVjxgxJ9porDw8PVatWTY0aNdLEiRPVoEEDvf322/xszycEoluYMUaRkZFasmSJ1q1bpypVqjhtb9Sokdzd3bV27VqrLSYmRrGxsQoLC5MkhYWFac+ePU4/YKKiouTj45PtB/it7HpzJf15ZqhDhw7y8PDQ0qVL5eXl5bSdufrTiy++qN27dys6OtpaJGnatGmaO3euJPvMlXT9+apcubJCQkKyPV5+6NAhhYaGSvpzvhITE7Vz505r+7p165SZmammTZvm/0EUkOvN1fnz5yVJbm7Ov5qKFStmnWmzy1zlJDMzU2lpafxszy+Feks3bspTTz1lfH19zYYNG0xcXJy1nD9/3urz5JNPmkqVKpl169aZHTt2mLCwMBMWFmZtz3o0s0OHDiY6OtqsWrXKlCtX7rZ7NPN6c5WUlGSaNm1q6tWrZ44cOeLU5/Lly8YY5uqvf66upKs8dn+7z5UxuZuvadOmGR8fH7N48WJz+PBhM2bMGOPl5WWOHDli9enYsaP5+9//brZv326+/fZbU7169dvuUfLrzdWlS5dMtWrVTMuWLc327dvNkSNHzFtvvWUcDodZvny5NY4d5urFF180GzduNMeOHTO7d+82L774onE4HGb16tXGGH625wcC0S1MUo7L3LlzrT4XLlwwTz/9tClTpowpUaKE6dGjh4mLi3Ma55dffjGdOnUy3t7eJiAgwIwcOdJ61Px2cb25Wr9+/VX7HDt2zBqHubr6Pn8NRMbYY66Myf18TZw40VSoUMGUKFHChIWFmc2bNzttP336tOnTp48pVaqU8fHxMY888og5d+5cAR5J/svNXB06dMj07NnTBAYGmhIlSpj69etnewzfDnP16KOPmtDQUOPh4WHKlStn2rVrZ4UhY/jZnh8cxhiTn2egAAAAijruIQIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAJQJP3yyy9yOBzWq0EAID8RiADkG4fDcc1l3LhxhV1ijo4cOaJHHnlEFSpUkKenp6pUqaI+ffpox44dBVoHoRAoOMULuwAAt6+4uDjr3xctWqSXX37Z6SWnpUqVKoyyrmnHjh1q166d6tatqw8++EA1a9bUuXPn9PXXX2vkyJHauHFjYZcIIB9whghAvgkODrYWX19fORwOaz0wMFBTp061zsI0bNhQq1atuupYGRkZevTRR1WzZk3FxsZKkr7++mvddddd8vLyUtWqVTV+/HhdvnzZ2sfhcOjDDz9Ujx49VKJECVWvXl1Lly696mcYYzRo0CBVr15dmzdvVpcuXXTHHXeoYcOGGjt2rL7++mur7549e3TvvffK29tbZcuW1ZAhQ5SSkmJtb9OmjYYPH+40fvfu3TVo0CBrvXLlynr99df16KOPqnTp0qpUqZJmz55tbc96G/zf//53ORwOtWnT5przDSDvCEQACsXbb7+tKVOm6K233tLu3bsVHh6uBx54QIcPH87WNy0tTQ899JCio6O1efNmVapUSZs3b9aAAQP0zDPPaP/+/frggw80b948vfbaa077jh8/Xv/4xz+0e/dude7cWf369dOZM2dyrCk6Olr79u3TyJEj5eaW/cejn5+fJCk1NVXh4eEqU6aMfvjhBy1evFhr1qxRZGTkDc/DlClTdPfdd2vXrl16+umn9dRTT1ln0b7//ntJ0po1axQXF6cvv/zyhscHkEuF/HJZADYxd+5c4+vra62HhISY1157zalP48aNzdNPP22MMebYsWNGktm8ebNp166dadGihUlMTLT6tmvXzrz++utO+//nP/8x5cuXt9YlmTFjxljrKSkpRpJZuXJljjUuWrTISDI//vjjNY9l9uzZpkyZMiYlJcVqW758uXFzczPx8fHGGGNat25tnnnmGaf9unXrZgYOHGith4aGmv79+1vrmZmZJjAw0MycOdNpDnbt2nXNegDcPO4hAlDgkpOTdeLECTVv3typvXnz5vrpp5+c2vr06aMKFSpo3bp18vb2ttp/+uknbdmyxemMUEZGhi5evKjz58+rRIkSkqT69etb20uWLCkfHx+dOnUqx7qMMbmq/8CBA2rQoIFKlizpVHtmZqZiYmIUFBSUq3GurC/rkuLV6gOQf7hkBqBI69y5s3bv3q1t27Y5taekpGj8+PGKjo62lj179ujw4cPy8vKy+rm7uzvt53A4lJmZmeNn3XnnnZKkgwcP3nTdbm5u2QJWenp6tn43Uh+A/EMgAlDgfHx8FBISoi1btji1b9myRbVr13Zqe+qpp/TGG2/ogQcecHrC66677lJMTIyqVauWbcnp/p/caNiwoWrXrq0pU6bkGEoSExMlSbVq1dJPP/2k1NRUp9rd3NxUo0YNSVK5cuWcnrLLyMjQ3r17b6geDw8Pa18A+YtABKBQjBo1Sm+++aYWLVqkmJgYvfjii4qOjtYzzzyTre/QoUP16quvqmvXrvr2228lSS+//LIWLFig8ePHa9++fTpw4IAWLlyoMWPG5Lkmh8OhuXPn6tChQ2rZsqVWrFihn3/+Wbt379Zrr72mbt26SZL69esnLy8vDRw4UHv37tX69es1dOhQPfzww9blsnvvvVfLly/X8uXLdfDgQT311FNWoMqtwMBAeXt7a9WqVTp58qSSkpLyfGwAro1ABKBQDBs2TCNGjNDIkSNVr149rVq1SkuXLlX16tVz7D98+HCNHz9enTt31tatWxUeHq5ly5Zp9erVaty4sZo1a6Zp06YpNDT0pupq0qSJduzYoWrVqunxxx9XrVq19MADD2jfvn2aPn26JKlEiRL65ptvdObMGTVu3FgPPvig2rVrp/fee88a59FHH9XAgQM1YMAAtW7dWlWrVlXbtm1vqJbixYvrnXfe0QcffKCQkBArkAFwPYfJ7V2EAAAAtynOEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANv7P02mbcQUBh7zAAAAAElFTkSuQmCC",
+      "text/plain": [
+       "<Figure size 640x480 with 1 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "313 193\n"
+     ]
+    }
+   ],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "token_dist = df['mistral_reasoning_prompt'].apply(lambda x: len(tokenizer.apply_chat_template(x)))\n",
+    "\n",
+    "# Plot the histogram of token counts\n",
+    "plt.hist(token_dist, bins=20, edgecolor='black')\n",
+    "plt.title(\"Histogram of Token Counts\")\n",
+    "plt.xlabel(\"Token Count\")\n",
+    "plt.ylabel(\"Frequency\")\n",
+    "plt.show()\n",
+    "print(max(token_dist), min(token_dist))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5a040685-4e01-4b78-9a04-3c4a4342f661",
+   "metadata": {},
+   "source": [
+    "# Deployment"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "8fa1a1a3-5bf3-463d-9ad2-f4a78c07388d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from huggingface_hub import create_inference_endpoint\n",
+    "from huggingface_hub import get_inference_endpoint\n",
+    "\n",
+    "\n",
+    "def get_my_endpoint():\n",
+    "    name = f\"poe-reasoning\"\n",
+    "    namespace=NAMESPACE\n",
+    "    try:\n",
+    "        endpoint = get_inference_endpoint(name, namespace=namespace)\n",
+    "        endpoint.wait()\n",
+    "    except:\n",
+    "        # Custom Docker image details\n",
+    "        custom_image = {\n",
+    "            \"health_route\": \"/health\",\n",
+    "            \"url\": \"ghcr.io/huggingface/text-generation-inference:2.4.1\",  # This is the min version\n",
+    "            \"env\": {\n",
+    "                # \"LORA_ADAPTERS\": \"derek-thomas/autotrain-mistral-v03-prompt-experiment-mc-FA-sg,derek-thomas/autotrain-mistral-v03-prompt-experiment-mc-FA-R-sg,derek-thomas/autotrain-mistral-v03-prompt-experiment-mc-FA-sg\",  # Add adapters here\n",
+    "                \"MAX_BATCH_PREFILL_TOKENS\": \"8192\",\n",
+    "                \"MAX_INPUT_TOKENS\": \"320\", # Set according to your needs\n",
+    "                \"MAX_TOTAL_TOKENS\": \"2000\", # Set according to your needs\n",
+    "                \"DISABLE_CUSTOM_KERNELS\": 'false',\n",
+    "                \"MODEL_ID\": \"/repository\"\n",
+    "            },\n",
+    "        }\n",
+    "        \n",
+    "        secrets = {\n",
+    "            \"HF_TOKEN\": get_token()\n",
+    "        }\n",
+    "        \n",
+    "        # Creating the inference endpoint\n",
+    "        endpoint = create_inference_endpoint(\n",
+    "            name=name,\n",
+    "            namespace=namespace,\n",
+    "            repository='mistralai/Mistral-7B-Instruct-v0.3',\n",
+    "            framework=\"pytorch\",\n",
+    "            accelerator=\"gpu\",\n",
+    "            instance_size=\"x1\",\n",
+    "            instance_type=\"nvidia-l4\",\n",
+    "            region=\"us-east-1\",\n",
+    "            vendor=\"aws\",\n",
+    "            min_replica=4,\n",
+    "            max_replica=4,\n",
+    "            task=\"text-generation\",\n",
+    "            custom_image=custom_image,\n",
+    "            secrets=secrets\n",
+    "        )\n",
+    "        # endpoint.wait()\n",
+    "            \n",
+    "    print(\"Your model is ready to use!\")\n",
+    "    endpoint.wait()\n",
+    "    return endpoint"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "0f2e2a06-6210-4cca-a96e-25fd2938d73d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Your model is ready to use!\n"
+     ]
+    }
+   ],
+   "source": [
+    "endpoint = get_my_endpoint()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "b3d76aa7-c97f-4e2d-9400-93b13f22b846",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'(a) Seconds and minutes: Satellite technology is not used for predicting seconds or minutes with precision. While atomic clocks on satellites help keep time for GPS, they are not used for predicting seconds or minutes in the general sense.\\n\\n(b) The strength and magnitude of an earthquake: While satellite technology can detect earthquakes based on changes in ground motion, it is not used to predict earthquakes accurately. The movements of tectonic plates are not predictable with sufficient certainty to allow for accurate earthquake prediction.\\n\\n(d) 70-75 degrees fahrenheit: Satellite technology is not used to predict temperature in this specific range. Satellites can help monitor and predict temperature variations globally, but they cannot target a specific temperature range, especially one as narrow as 70-75°F.\\n\\n(e) Rapid changes occur: Satellite technology is not applicable to predict rapid changes in general, as they may refer to various phenomena, not just those related to satellite-monitored data.\\n\\n(f) Dead-ends and false starts: Satellite technology does not provide information on dead-ends and false starts. These concepts are related to navigation or decision-making processes, not satellite observations.\\n\\n(h) Around 5 to 27 degrees celsius: Just as with the Fahrenheit example, satellites are not used to predict a specific temperature range like this. Satellite technology can help predict temperature variations, but it does not target a specific range of temperatures.\\n\\nNow, we are left with option (c) \"What it\\'s like outside each day.\"\\n\\nSatellite technology is extensively used in monitoring and predicting weather patterns and providing us with up-to-date information about our weather conditions. This includes cloud cover, rainfall, temperature, and wind speeds. In this way, we can understand what it\\'s like outside on any given day.'"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "response = endpoint.client.chat_completion(df['mistral_reasoning_prompt'].iloc[0], max_tokens=1650)\n",
+    "response.choices[0].message.content"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "7d3f0277-8b3a-4171-bcb5-b8bda5d5f242",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "8e2aee49e93d4df492e32706b9a0b6f4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Processing Prompts:   0%|          | 0/8413 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import nest_asyncio\n",
+    "import asyncio\n",
+    "import pandas as pd\n",
+    "from concurrent.futures import ThreadPoolExecutor\n",
+    "from tqdm.notebook import tqdm\n",
+    "\n",
+    "# Assuming 'endpoint' and 'df' are already defined\n",
+    "\n",
+    "nest_asyncio.apply()\n",
+    "\n",
+    "async def async_chat_completion(prompt, max_tokens=1650):\n",
+    "    response = await endpoint.async_client.chat_completion(prompt, max_tokens=max_tokens)\n",
+    "    return response.choices[0].message.content\n",
+    "\n",
+    "async def generate_mistral_reasoning(prompts, max_tokens=1650, num_workers=64):\n",
+    "    loop = asyncio.get_event_loop()\n",
+    "    with ThreadPoolExecutor(max_workers=num_workers) as executor:\n",
+    "        tasks = [loop.run_in_executor(executor, lambda p=prompt: asyncio.run(async_chat_completion(p, max_tokens))) for prompt in prompts]\n",
+    "        results = []\n",
+    "        with tqdm(total=len(tasks), desc=\"Processing Prompts\") as pbar:\n",
+    "            for result in asyncio.as_completed(tasks):\n",
+    "                results.append(await result)\n",
+    "                pbar.update(1)\n",
+    "        return results\n",
+    "\n",
+    "prompts = df['mistral_reasoning_prompt'].tolist()\n",
+    "mistral_reasonings = await generate_mistral_reasoning(prompts)\n",
+    "df['mistral_reasoning'] = mistral_reasonings\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "573008db-78aa-4ca9-992b-3bdebcfd5fc9",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "InferenceEndpoint(name='poe-reasoning', namespace='HF-test-lab', repository='mistralai/Mistral-7B-Instruct-v0.3', status='paused', url=None)"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "endpoint.pause()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "0a31bb96-66cd-4f03-83b4-25b64c58a591",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0       Incorrect answers and explanations:\\n\\n1. Elec...\n",
+       "1       Sure, let's examine each answer and justify wh...\n",
+       "2       Sure, let's go through each of the provided an...\n",
+       "3       1. Reading the question carefully, we can see ...\n",
+       "4       1. Food: While essential for the growth and he...\n",
+       "                              ...                        \n",
+       "8408    1. Read the question and options carefully: Th...\n",
+       "8409    1. Read the question and options carefully: Th...\n",
+       "8410    1. Read the question and options carefully: Th...\n",
+       "8411    1. First, let's read the question and options ...\n",
+       "8412    1. Read the question and options carefully: Th...\n",
+       "Name: mistral_reasoning, Length: 8413, dtype: object"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.mistral_reasoning"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "27542a33-187e-4b3f-aa20-4177f778312d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>formatted_question</th>\n",
+       "      <th>combined_fact</th>\n",
+       "      <th>answer_key</th>\n",
+       "      <th>topic</th>\n",
+       "      <th>explanation</th>\n",
+       "      <th>question_text</th>\n",
+       "      <th>answer_choices</th>\n",
+       "      <th>mistral_reasoning_prompt</th>\n",
+       "      <th>mistral_reasoning</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>what is satellite technology used for predicti...</td>\n",
+       "      <td>satellite technology is used for predicting wh...</td>\n",
+       "      <td>c</td>\n",
+       "      <td>Technology</td>\n",
+       "      <td>a) Seconds and minutes: This option is incorre...</td>\n",
+       "      <td>What is satellite technology used for predicting?</td>\n",
+       "      <td>(a) Seconds and minutes (b) The strength and m...</td>\n",
+       "      <td>[{'role': 'user', 'content': 'You are an AI as...</td>\n",
+       "      <td>Incorrect answers and explanations:\\n\\n1. Elec...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>what does irradiating food do? (a) relieve pai...</td>\n",
+       "      <td>irradiated food improves food safety.</td>\n",
+       "      <td>c</td>\n",
+       "      <td>Food science</td>\n",
+       "      <td>(a) Relieve pain: This option is not correct b...</td>\n",
+       "      <td>What does irradiating food do?</td>\n",
+       "      <td>(a) Relieve pain (b) Enhance food's nutrients ...</td>\n",
+       "      <td>[{'role': 'user', 'content': 'You are an AI as...</td>\n",
+       "      <td>Sure, let's examine each answer and justify wh...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>what protects a mammal's skin? (a) fiber folli...</td>\n",
+       "      <td>fiber follicles protect mammal skin</td>\n",
+       "      <td>a</td>\n",
+       "      <td>Biology</td>\n",
+       "      <td>b) Exfoliation: Exfoliation is the process of ...</td>\n",
+       "      <td>What protects a mammal's skin?</td>\n",
+       "      <td>(a) Fiber follicles (b) Exfoliation (c) Resist...</td>\n",
+       "      <td>[{'role': 'user', 'content': 'You are an AI as...</td>\n",
+       "      <td>Sure, let's go through each of the provided an...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>what do earthworms do when a segment breaks of...</td>\n",
+       "      <td>earthworms can regrow segments that break off</td>\n",
+       "      <td>b</td>\n",
+       "      <td>Biology</td>\n",
+       "      <td>a) Dies: This option is not correct because ea...</td>\n",
+       "      <td>What do earthworms do when a segment breaks off?</td>\n",
+       "      <td>(a) Dies (b) Regrows it (c) Reproduces (d) Sed...</td>\n",
+       "      <td>[{'role': 'user', 'content': 'You are an AI as...</td>\n",
+       "      <td>1. Reading the question carefully, we can see ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>lightning can be bad for what? (a) the environ...</td>\n",
+       "      <td>lightning can be bad for the environment.</td>\n",
+       "      <td>a</td>\n",
+       "      <td>Electricity</td>\n",
+       "      <td>b) Rainstorms: Lightning is actually a natural...</td>\n",
+       "      <td>Lightning can be bad for what?</td>\n",
+       "      <td>(a) The environment (b) Rainstorms (c) Destruc...</td>\n",
+       "      <td>[{'role': 'user', 'content': 'You are an AI as...</td>\n",
+       "      <td>1. Food: While essential for the growth and he...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8408</th>\n",
+       "      <td>organisms that can cause infection do what? (a...</td>\n",
+       "      <td>organisms that can cause infection make humans...</td>\n",
+       "      <td>g</td>\n",
+       "      <td>Biology</td>\n",
+       "      <td>a) Bandaging open sores is not the correct ans...</td>\n",
+       "      <td>Organisms that can cause infection do what?</td>\n",
+       "      <td>(a) Bandage open sores (b) Keep flesh clean (c...</td>\n",
+       "      <td>[{'role': 'user', 'content': 'You are an AI as...</td>\n",
+       "      <td>1. Read the question and options carefully: Th...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8409</th>\n",
+       "      <td>fungi are living things that cannot make thei...</td>\n",
+       "      <td>fungi are living things that cannot make their...</td>\n",
+       "      <td>a</td>\n",
+       "      <td>Biology</td>\n",
+       "      <td>b) Fungi are living things that can make their...</td>\n",
+       "      <td>Fungi are living things that cannot make their...</td>\n",
+       "      <td>(a) Food (b) Cells (c) Energy (d) Fruits (e) H...</td>\n",
+       "      <td>[{'role': 'user', 'content': 'You are an AI as...</td>\n",
+       "      <td>1. Read the question and options carefully: Th...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8410</th>\n",
+       "      <td>an overheated body can use water for: (a) meta...</td>\n",
+       "      <td>the evaporation of water from the skin cools t...</td>\n",
+       "      <td>g</td>\n",
+       "      <td>Biology</td>\n",
+       "      <td>a) Metabolic reaction: This option is incorrec...</td>\n",
+       "      <td>An overheated body can use water for:?</td>\n",
+       "      <td>(a) Metabolic reaction (b) Dehydrating (c) Rai...</td>\n",
+       "      <td>[{'role': 'user', 'content': 'You are an AI as...</td>\n",
+       "      <td>1. Read the question and options carefully: Th...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8411</th>\n",
+       "      <td>what is essential for cellular respiration for...</td>\n",
+       "      <td>plants are essential for cellular respiration ...</td>\n",
+       "      <td>f</td>\n",
+       "      <td>Biology</td>\n",
+       "      <td>a) Electrons are involved in cellular respirat...</td>\n",
+       "      <td>What is essential for cellular respiration for...</td>\n",
+       "      <td>(a) Electron (b) Glucose (c) Energy (d) Energy...</td>\n",
+       "      <td>[{'role': 'user', 'content': 'You are an AI as...</td>\n",
+       "      <td>1. First, let's read the question and options ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8412</th>\n",
+       "      <td>what helps insulate and protect the body? (a) ...</td>\n",
+       "      <td>living cells in follicles help insulate and pr...</td>\n",
+       "      <td>b</td>\n",
+       "      <td>Biology</td>\n",
+       "      <td>a) H2O: Water is essential for life, but it do...</td>\n",
+       "      <td>What helps insulate and protect the body?</td>\n",
+       "      <td>(a) H2o (b) Living cells in follicles (c) Laye...</td>\n",
+       "      <td>[{'role': 'user', 'content': 'You are an AI as...</td>\n",
+       "      <td>1. Read the question and options carefully: Th...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>8413 rows × 9 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                     formatted_question  \\\n",
+       "0     what is satellite technology used for predicti...   \n",
+       "1     what does irradiating food do? (a) relieve pai...   \n",
+       "2     what protects a mammal's skin? (a) fiber folli...   \n",
+       "3     what do earthworms do when a segment breaks of...   \n",
+       "4     lightning can be bad for what? (a) the environ...   \n",
+       "...                                                 ...   \n",
+       "8408  organisms that can cause infection do what? (a...   \n",
+       "8409   fungi are living things that cannot make thei...   \n",
+       "8410  an overheated body can use water for: (a) meta...   \n",
+       "8411  what is essential for cellular respiration for...   \n",
+       "8412  what helps insulate and protect the body? (a) ...   \n",
+       "\n",
+       "                                          combined_fact answer_key  \\\n",
+       "0     satellite technology is used for predicting wh...          c   \n",
+       "1                 irradiated food improves food safety.          c   \n",
+       "2                   fiber follicles protect mammal skin          a   \n",
+       "3         earthworms can regrow segments that break off          b   \n",
+       "4             lightning can be bad for the environment.          a   \n",
+       "...                                                 ...        ...   \n",
+       "8408  organisms that can cause infection make humans...          g   \n",
+       "8409  fungi are living things that cannot make their...          a   \n",
+       "8410  the evaporation of water from the skin cools t...          g   \n",
+       "8411  plants are essential for cellular respiration ...          f   \n",
+       "8412  living cells in follicles help insulate and pr...          b   \n",
+       "\n",
+       "             topic                                        explanation  \\\n",
+       "0       Technology  a) Seconds and minutes: This option is incorre...   \n",
+       "1     Food science  (a) Relieve pain: This option is not correct b...   \n",
+       "2          Biology  b) Exfoliation: Exfoliation is the process of ...   \n",
+       "3          Biology  a) Dies: This option is not correct because ea...   \n",
+       "4      Electricity  b) Rainstorms: Lightning is actually a natural...   \n",
+       "...            ...                                                ...   \n",
+       "8408       Biology  a) Bandaging open sores is not the correct ans...   \n",
+       "8409       Biology  b) Fungi are living things that can make their...   \n",
+       "8410       Biology  a) Metabolic reaction: This option is incorrec...   \n",
+       "8411       Biology  a) Electrons are involved in cellular respirat...   \n",
+       "8412       Biology  a) H2O: Water is essential for life, but it do...   \n",
+       "\n",
+       "                                          question_text  \\\n",
+       "0     What is satellite technology used for predicting?   \n",
+       "1                        What does irradiating food do?   \n",
+       "2                        What protects a mammal's skin?   \n",
+       "3      What do earthworms do when a segment breaks off?   \n",
+       "4                        Lightning can be bad for what?   \n",
+       "...                                                 ...   \n",
+       "8408        Organisms that can cause infection do what?   \n",
+       "8409  Fungi are living things that cannot make their...   \n",
+       "8410             An overheated body can use water for:?   \n",
+       "8411  What is essential for cellular respiration for...   \n",
+       "8412          What helps insulate and protect the body?   \n",
+       "\n",
+       "                                         answer_choices  \\\n",
+       "0     (a) Seconds and minutes (b) The strength and m...   \n",
+       "1     (a) Relieve pain (b) Enhance food's nutrients ...   \n",
+       "2     (a) Fiber follicles (b) Exfoliation (c) Resist...   \n",
+       "3     (a) Dies (b) Regrows it (c) Reproduces (d) Sed...   \n",
+       "4     (a) The environment (b) Rainstorms (c) Destruc...   \n",
+       "...                                                 ...   \n",
+       "8408  (a) Bandage open sores (b) Keep flesh clean (c...   \n",
+       "8409  (a) Food (b) Cells (c) Energy (d) Fruits (e) H...   \n",
+       "8410  (a) Metabolic reaction (b) Dehydrating (c) Rai...   \n",
+       "8411  (a) Electron (b) Glucose (c) Energy (d) Energy...   \n",
+       "8412  (a) H2o (b) Living cells in follicles (c) Laye...   \n",
+       "\n",
+       "                               mistral_reasoning_prompt  \\\n",
+       "0     [{'role': 'user', 'content': 'You are an AI as...   \n",
+       "1     [{'role': 'user', 'content': 'You are an AI as...   \n",
+       "2     [{'role': 'user', 'content': 'You are an AI as...   \n",
+       "3     [{'role': 'user', 'content': 'You are an AI as...   \n",
+       "4     [{'role': 'user', 'content': 'You are an AI as...   \n",
+       "...                                                 ...   \n",
+       "8408  [{'role': 'user', 'content': 'You are an AI as...   \n",
+       "8409  [{'role': 'user', 'content': 'You are an AI as...   \n",
+       "8410  [{'role': 'user', 'content': 'You are an AI as...   \n",
+       "8411  [{'role': 'user', 'content': 'You are an AI as...   \n",
+       "8412  [{'role': 'user', 'content': 'You are an AI as...   \n",
+       "\n",
+       "                                      mistral_reasoning  \n",
+       "0     Incorrect answers and explanations:\\n\\n1. Elec...  \n",
+       "1     Sure, let's examine each answer and justify wh...  \n",
+       "2     Sure, let's go through each of the provided an...  \n",
+       "3     1. Reading the question carefully, we can see ...  \n",
+       "4     1. Food: While essential for the growth and he...  \n",
+       "...                                                 ...  \n",
+       "8408  1. Read the question and options carefully: Th...  \n",
+       "8409  1. Read the question and options carefully: Th...  \n",
+       "8410  1. Read the question and options carefully: Th...  \n",
+       "8411  1. First, let's read the question and options ...  \n",
+       "8412  1. Read the question and options carefully: Th...  \n",
+       "\n",
+       "[8413 rows x 9 columns]"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "55bc6940-07e4-402f-9c56-fb830a35290f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b9047ef7088b4eaa83916017c17c0bbc",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "2bc167b9d94d4b8588f71c5f7f82722e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Creating parquet from Arrow format:   0%|          | 0/9 [00:00<?, ?ba/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "CommitInfo(commit_url='https://huggingface.co/datasets/derek-thomas/labeled-multiple-choice-explained-mistral-reasoning/commit/f6554f7b3e0671558956eed6a42453b281b6f7e8', commit_message='Upload dataset', commit_description='', oid='f6554f7b3e0671558956eed6a42453b281b6f7e8', pr_url=None, repo_url=RepoUrl('https://huggingface.co/datasets/derek-thomas/labeled-multiple-choice-explained-mistral-reasoning', endpoint='https://huggingface.co', repo_type='dataset', repo_id='derek-thomas/labeled-multiple-choice-explained-mistral-reasoning'), pr_revision=None, pr_num=None)"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from datasets import Dataset\n",
+    "\n",
+    "dataset = Dataset.from_pandas(df)\n",
+    "dataset.push_to_hub(OUTPUT_DATASET)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "24857480-8ecc-4dc9-a5b0-af6debca162d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

01-poe-dataset-creation.ipynb

@@ -0,0 +1,1421 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "68e9310f-109d-4f30-b263-d1e6c058ee80",
+   "metadata": {},
+   "source": [
+    "# Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "6805b3b5-782b-437c-82b3-9392abb5a599",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# %pip install -q -r requirements.txt"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "94f0fcdd-1653-440e-8ebc-9c33d931163a",
+   "metadata": {},
+   "source": [
+    "## Config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "5d0bd22f-293e-4c15-9dfe-8070553f42b5",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "INPUT_DATASET = 'derek-thomas/labeled-multiple-choice-explained-mistral-reasoning'\n",
+    "REVISION = '536f3b8'\n",
+    "OUTPUT_DATASET = 'derek-thomas/labeled-multiple-choice-explained-mistral-tokenized'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "a1fc7a29-6b60-446d-b708-012f897de6a9",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "85ca07b896804801b5ccd629341e965c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from transformers import AutoTokenizer\n",
+    "from huggingface_hub import get_token, login\n",
+    "\n",
+    "login()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "f17edf80-7318-4b65-ae60-7433a12ad8cb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "BASE_MODEL = 'mistralai/Mistral-7B-Instruct-v0.3'\n",
+    "tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, token=get_token())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "df697715-3cff-4177-bfa0-348a76d00a73",
+   "metadata": {},
+   "source": [
+    "# Prompt Experiment\n",
+    "## Goal\n",
+    "I want to explore a few scenarios for prompt fine-tuning. Lets consider a scenario where we have the following available:\n",
+    "- (Q) Question\n",
+    "- (AC) Answer Choices\n",
+    "- (R) Reasoning\n",
+    "- (FA) Final Answer\n",
+    "\n",
+    "How would performance differ if we tried changing the order?\n",
+    "\n",
+    "Scenario 1: `Q - AC - R - FA`\n",
+    "This is the most natural, we want the model to generate reasoning before the final answer. Based on how decoding works, this will give the most information before selecting the Final Answer.\n",
+    "\n",
+    "Scenario 2: `Q - AC - FA - R`\n",
+    "This is quite awkward. Why would we put our reasoning last? Its faster. Could the model be trained in such a way to \"know\" the reasoning before responding with it? If so we could save a lot of tokens. Im skeptical, but its worth testing.\n",
+    "\n",
+    "Scenario 3: `Q - AC - FA`\n",
+    "This is our fine-tuning control.\n",
+    "\n",
+    "Scenario 4: Base\n",
+    "This is our un-fine-tuned control.\n",
+    "\n",
+    "In each of these scenarios I will build prompts with strucutred generation to fine-tune with. I noticed some difficulty in a first pass with getting consistent response formats, but thats out of scope, so structured generation can help a lot here.\n",
+    "\n",
+    "Datasets wont store complex structures like lists of dicts of different types (needed for structured generation, so its easiest if I tokenize. Ill be using Mistral, so Ill skip the system prompt. Its simple enough to come back and change this for a different model in this notebook.\n",
+    "\n",
+    "## Implementation\n",
+    "To explore this goal, we will start with [layoric/labeled-multiple-choice-explained](https://huggingface.co/datasets/layoric/labeled-multiple-choice-explained) as our dataset. It has explanations already provided by GPT-3.5-turbo. Given that these explanations are a bit different than what mistral would do, it might be useful if we generate some from mistral as well. Based on [this notebook](./poe-generate-mistral-reasoning.ipynb) we have been able to generate mistral reasoning in this refined dataset [derek-thomas/labeled-multiple-choice-explained-mistral-reasoning](https://huggingface.co/datasets/derek-thomas/labeled-multiple-choice-explained-mistral-reasoning).\n",
+    "\n",
+    "In this notebook we will format our data such that we can try each experiment and then we will push it to my repo: [derek-thomas/labeled-multiple-choice-explained](https://huggingface.co/datasets/derek-thomas/labeled-multiple-choice-explained)."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "13cc5c85-0642-4348-8167-60b5ed5d37d5",
+   "metadata": {},
+   "source": [
+    "## Imports"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "7925796f-6b90-4c43-8c22-e3906cfbdf3b",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from datasets import load_dataset\n",
+    "import json"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c5750868-8442-4d0b-9259-4e7e490793bb",
+   "metadata": {},
+   "source": [
+    "## Load and Preprocess the Dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "d8d7a40a-57af-45b9-bcea-8ac4c3ec0e61",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Before Cleaning: 8413 rows\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>formatted_question</th>\n",
+       "      <th>combined_fact</th>\n",
+       "      <th>answer_key</th>\n",
+       "      <th>topic</th>\n",
+       "      <th>gpt3_5_reasoning</th>\n",
+       "      <th>question_text</th>\n",
+       "      <th>answer_choices</th>\n",
+       "      <th>mistral_reasoning</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>what is satellite technology used for predicti...</td>\n",
+       "      <td>satellite technology is used for predicting wh...</td>\n",
+       "      <td>c</td>\n",
+       "      <td>Technology</td>\n",
+       "      <td>a) Seconds and minutes: This option is incorre...</td>\n",
+       "      <td>What is satellite technology used for predicting?</td>\n",
+       "      <td>(a) Seconds and minutes (b) The strength and m...</td>\n",
+       "      <td>Incorrect answers and explanations:\\n\\n1. Elec...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>what does irradiating food do? (a) relieve pai...</td>\n",
+       "      <td>irradiated food improves food safety.</td>\n",
+       "      <td>c</td>\n",
+       "      <td>Food science</td>\n",
+       "      <td>(a) Relieve pain: This option is not correct b...</td>\n",
+       "      <td>What does irradiating food do?</td>\n",
+       "      <td>(a) Relieve pain (b) Enhance food's nutrients ...</td>\n",
+       "      <td>Sure, let's examine each answer and justify wh...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>what protects a mammal's skin? (a) fiber folli...</td>\n",
+       "      <td>fiber follicles protect mammal skin</td>\n",
+       "      <td>a</td>\n",
+       "      <td>Biology</td>\n",
+       "      <td>b) Exfoliation: Exfoliation is the process of ...</td>\n",
+       "      <td>What protects a mammal's skin?</td>\n",
+       "      <td>(a) Fiber follicles (b) Exfoliation (c) Resist...</td>\n",
+       "      <td>Sure, let's go through each of the provided an...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>what do earthworms do when a segment breaks of...</td>\n",
+       "      <td>earthworms can regrow segments that break off</td>\n",
+       "      <td>b</td>\n",
+       "      <td>Biology</td>\n",
+       "      <td>a) Dies: This option is not correct because ea...</td>\n",
+       "      <td>What do earthworms do when a segment breaks off?</td>\n",
+       "      <td>(a) Dies (b) Regrows it (c) Reproduces (d) Sed...</td>\n",
+       "      <td>1. Reading the question carefully, we can see ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>lightning can be bad for what? (a) the environ...</td>\n",
+       "      <td>lightning can be bad for the environment.</td>\n",
+       "      <td>a</td>\n",
+       "      <td>Electricity</td>\n",
+       "      <td>b) Rainstorms: Lightning is actually a natural...</td>\n",
+       "      <td>Lightning can be bad for what?</td>\n",
+       "      <td>(a) The environment (b) Rainstorms (c) Destruc...</td>\n",
+       "      <td>1. Food: While essential for the growth and he...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8408</th>\n",
+       "      <td>organisms that can cause infection do what? (a...</td>\n",
+       "      <td>organisms that can cause infection make humans...</td>\n",
+       "      <td>g</td>\n",
+       "      <td>Biology</td>\n",
+       "      <td>a) Bandaging open sores is not the correct ans...</td>\n",
+       "      <td>Organisms that can cause infection do what?</td>\n",
+       "      <td>(a) Bandage open sores (b) Keep flesh clean (c...</td>\n",
+       "      <td>1. Read the question and options carefully: Th...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8409</th>\n",
+       "      <td>fungi are living things that cannot make thei...</td>\n",
+       "      <td>fungi are living things that cannot make their...</td>\n",
+       "      <td>a</td>\n",
+       "      <td>Biology</td>\n",
+       "      <td>b) Fungi are living things that can make their...</td>\n",
+       "      <td>Fungi are living things that cannot make their...</td>\n",
+       "      <td>(a) Food (b) Cells (c) Energy (d) Fruits (e) H...</td>\n",
+       "      <td>1. Read the question and options carefully: Th...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8410</th>\n",
+       "      <td>an overheated body can use water for: (a) meta...</td>\n",
+       "      <td>the evaporation of water from the skin cools t...</td>\n",
+       "      <td>g</td>\n",
+       "      <td>Biology</td>\n",
+       "      <td>a) Metabolic reaction: This option is incorrec...</td>\n",
+       "      <td>An overheated body can use water for:?</td>\n",
+       "      <td>(a) Metabolic reaction (b) Dehydrating (c) Rai...</td>\n",
+       "      <td>1. Read the question and options carefully: Th...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8411</th>\n",
+       "      <td>what is essential for cellular respiration for...</td>\n",
+       "      <td>plants are essential for cellular respiration ...</td>\n",
+       "      <td>f</td>\n",
+       "      <td>Biology</td>\n",
+       "      <td>a) Electrons are involved in cellular respirat...</td>\n",
+       "      <td>What is essential for cellular respiration for...</td>\n",
+       "      <td>(a) Electron (b) Glucose (c) Energy (d) Energy...</td>\n",
+       "      <td>1. First, let's read the question and options ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8412</th>\n",
+       "      <td>what helps insulate and protect the body? (a) ...</td>\n",
+       "      <td>living cells in follicles help insulate and pr...</td>\n",
+       "      <td>b</td>\n",
+       "      <td>Biology</td>\n",
+       "      <td>a) H2O: Water is essential for life, but it do...</td>\n",
+       "      <td>What helps insulate and protect the body?</td>\n",
+       "      <td>(a) H2o (b) Living cells in follicles (c) Laye...</td>\n",
+       "      <td>1. Read the question and options carefully: Th...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>8413 rows × 8 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                     formatted_question  \\\n",
+       "0     what is satellite technology used for predicti...   \n",
+       "1     what does irradiating food do? (a) relieve pai...   \n",
+       "2     what protects a mammal's skin? (a) fiber folli...   \n",
+       "3     what do earthworms do when a segment breaks of...   \n",
+       "4     lightning can be bad for what? (a) the environ...   \n",
+       "...                                                 ...   \n",
+       "8408  organisms that can cause infection do what? (a...   \n",
+       "8409   fungi are living things that cannot make thei...   \n",
+       "8410  an overheated body can use water for: (a) meta...   \n",
+       "8411  what is essential for cellular respiration for...   \n",
+       "8412  what helps insulate and protect the body? (a) ...   \n",
+       "\n",
+       "                                          combined_fact answer_key  \\\n",
+       "0     satellite technology is used for predicting wh...          c   \n",
+       "1                 irradiated food improves food safety.          c   \n",
+       "2                   fiber follicles protect mammal skin          a   \n",
+       "3         earthworms can regrow segments that break off          b   \n",
+       "4             lightning can be bad for the environment.          a   \n",
+       "...                                                 ...        ...   \n",
+       "8408  organisms that can cause infection make humans...          g   \n",
+       "8409  fungi are living things that cannot make their...          a   \n",
+       "8410  the evaporation of water from the skin cools t...          g   \n",
+       "8411  plants are essential for cellular respiration ...          f   \n",
+       "8412  living cells in follicles help insulate and pr...          b   \n",
+       "\n",
+       "             topic                                   gpt3_5_reasoning  \\\n",
+       "0       Technology  a) Seconds and minutes: This option is incorre...   \n",
+       "1     Food science  (a) Relieve pain: This option is not correct b...   \n",
+       "2          Biology  b) Exfoliation: Exfoliation is the process of ...   \n",
+       "3          Biology  a) Dies: This option is not correct because ea...   \n",
+       "4      Electricity  b) Rainstorms: Lightning is actually a natural...   \n",
+       "...            ...                                                ...   \n",
+       "8408       Biology  a) Bandaging open sores is not the correct ans...   \n",
+       "8409       Biology  b) Fungi are living things that can make their...   \n",
+       "8410       Biology  a) Metabolic reaction: This option is incorrec...   \n",
+       "8411       Biology  a) Electrons are involved in cellular respirat...   \n",
+       "8412       Biology  a) H2O: Water is essential for life, but it do...   \n",
+       "\n",
+       "                                          question_text  \\\n",
+       "0     What is satellite technology used for predicting?   \n",
+       "1                        What does irradiating food do?   \n",
+       "2                        What protects a mammal's skin?   \n",
+       "3      What do earthworms do when a segment breaks off?   \n",
+       "4                        Lightning can be bad for what?   \n",
+       "...                                                 ...   \n",
+       "8408        Organisms that can cause infection do what?   \n",
+       "8409  Fungi are living things that cannot make their...   \n",
+       "8410             An overheated body can use water for:?   \n",
+       "8411  What is essential for cellular respiration for...   \n",
+       "8412          What helps insulate and protect the body?   \n",
+       "\n",
+       "                                         answer_choices  \\\n",
+       "0     (a) Seconds and minutes (b) The strength and m...   \n",
+       "1     (a) Relieve pain (b) Enhance food's nutrients ...   \n",
+       "2     (a) Fiber follicles (b) Exfoliation (c) Resist...   \n",
+       "3     (a) Dies (b) Regrows it (c) Reproduces (d) Sed...   \n",
+       "4     (a) The environment (b) Rainstorms (c) Destruc...   \n",
+       "...                                                 ...   \n",
+       "8408  (a) Bandage open sores (b) Keep flesh clean (c...   \n",
+       "8409  (a) Food (b) Cells (c) Energy (d) Fruits (e) H...   \n",
+       "8410  (a) Metabolic reaction (b) Dehydrating (c) Rai...   \n",
+       "8411  (a) Electron (b) Glucose (c) Energy (d) Energy...   \n",
+       "8412  (a) H2o (b) Living cells in follicles (c) Laye...   \n",
+       "\n",
+       "                                      mistral_reasoning  \n",
+       "0     Incorrect answers and explanations:\\n\\n1. Elec...  \n",
+       "1     Sure, let's examine each answer and justify wh...  \n",
+       "2     Sure, let's go through each of the provided an...  \n",
+       "3     1. Reading the question carefully, we can see ...  \n",
+       "4     1. Food: While essential for the growth and he...  \n",
+       "...                                                 ...  \n",
+       "8408  1. Read the question and options carefully: Th...  \n",
+       "8409  1. Read the question and options carefully: Th...  \n",
+       "8410  1. Read the question and options carefully: Th...  \n",
+       "8411  1. First, let's read the question and options ...  \n",
+       "8412  1. Read the question and options carefully: Th...  \n",
+       "\n",
+       "[8413 rows x 8 columns]"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Load dataset from Hugging Face Hub\n",
+    "dataset = load_dataset(INPUT_DATASET, split='train')\n",
+    "\n",
+    "# Convert to pandas dataframe\n",
+    "df = dataset.to_pandas()\n",
+    "print(f\"Before Cleaning: {len(df)} rows\")\n",
+    "\n",
+    "# Drop the __index_level_0__ column if it exists\n",
+    "df.drop(columns=['mistral_reasoning_prompt'], errors='ignore', inplace=True)\n",
+    "\n",
+    "# Ensure all values in 'formatted_question' are strings\n",
+    "df.rename(columns={\n",
+    "    'explanation': 'gpt3_5_reasoning',\n",
+    "}, inplace=True)\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2511bc04-f611-4dc7-b3ed-e477907b0200",
+   "metadata": {},
+   "source": [
+    "## Create Prompts from Processed Data"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d124c7cf-a369-46a9-94db-069894145959",
+   "metadata": {},
+   "source": [
+    "We need to convert our sample into a format similar to below for each of the scenarios.\n",
+    "\n",
+    "```\n",
+    "[\n",
+    "    {\"content\": user_content, \"role\": \"user\"},\n",
+    "    {\"content\": assistant_response, \"role\": \"assistant\"}\n",
+    "]\n",
+    "```\n",
+    "\n",
+    "We should include a helpful system_prompt with a general trivia prefix, and a suffix that contains instructions that fit each scenario.\n",
+    "The `user_content` will have the Question and answer choices.\n",
+    "The `assistant_response` should reflect the scenario. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "1c6554a6-4717-4bf0-ae51-102630d40fd7",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "df['user_prompt'] = df.apply(lambda row: f\"Question: {row['question_text']}\\nAnswer Choices: {row['answer_choices']}\", axis=1)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1391b7e1-2462-41e3-b2f0-5a8f6c95859b",
+   "metadata": {},
+   "source": [
+    "Here we need to create the structure of our conversation. Each system prompt should reflect the instructions we want, so we can start with a prefix and add in the specifics for each scenario."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2c817304-9ecc-43d9-83b9-5bb8dd662797",
+   "metadata": {},
+   "source": [
+    "### Reasoning Final Answer Structured Generation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "878727dc-4801-4376-be26-1cc601cb5f92",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<s>[INST] Answer the Question and include your Reasoning and the Final Answer in a json like: {\"Reasoning: \"...\", \"Final Answer\": \"x\"} where x is a letter that corresponds to the answer choice which is a letter between a and h.\n",
+      "Question: What is satellite technology used for predicting?\n",
+      "Answer Choices: (a) Seconds and minutes (b) The strength and magnitude of an earthquake (c) What it's like outside each day (d) 70-75 degrees fahrenheit (e) Rapid changes occur (f) Dead-ends and false starts. (g) Snow, ice, and rock (h) Around 5 to 27 degrees celsius[/INST] {'Reasoning': \"a) Seconds and minutes: This option is incorrect because satellite technology is not used for predicting time intervals. Satellite technology is used for various purposes such as communication, navigation, and weather forecasting, but it is not used for predicting time intervals.\\n\\nb) The strength and magnitude of an earthquake: This option is incorrect because satellite technology is not used for predicting earthquakes. Earthquake prediction is a complex process that involves seismology and other scientific methods, but satellite technology is not one of them.\\n\\nd) 70-75 degrees Fahrenheit: This option is incorrect because satellite technology is not used for predicting specific temperature ranges. While satellite technology can provide temperature data, it is not used for predicting specific temperature ranges.\\n\\ne) Rapid changes occur: This option is too vague and does not provide enough information to determine whether it is correct or not. Satellite technology can be used to monitor changes in various environmental factors, but it is not used specifically for predicting rapid changes.\\n\\nf) Dead-ends and false starts: This option is incorrect because it is not related to satellite technology or any type of prediction.\\n\\ng) Snow, ice, and rock: This option is incorrect because it is too specific and does not cover the broad range of predictions that satellite technology can be used for. While satellite technology can be used to monitor snow, ice, and rock formations, it is not used exclusively for this purpose.\\n\\nh) Around 5 to 27 degrees Celsius: This option is incorrect because it is too specific and does not cover the broad range of temperature predictions that satellite technology can be used for. While satellite technology can provide temperature data, it is not used exclusively for predicting temperatures within a specific range.\\n\\nTherefore, the correct answer is c) what it's like outside each day, as satellite technology is commonly used for weather forecasting and predicting daily weather conditions.\", 'Final Answer': 'c'}</s>\n",
+      "\n",
+      "\n",
+      "\n",
+      "<s>[INST] Answer the Question and include your Reasoning and the Final Answer in a json like: {\"Reasoning: \"...\", \"Final Answer\": \"x\"} where x is a letter that corresponds to the answer choice which is a letter between a and h.\n",
+      "Question: What is satellite technology used for predicting?\n",
+      "Answer Choices: (a) Seconds and minutes (b) The strength and magnitude of an earthquake (c) What it's like outside each day (d) 70-75 degrees fahrenheit (e) Rapid changes occur (f) Dead-ends and false starts. (g) Snow, ice, and rock (h) Around 5 to 27 degrees celsius[/INST] {'Reasoning': 'Incorrect answers and explanations:\\n\\n1. Electrical energy: Cells are not visible with electrical energy. Cells are physical structures and need to be observed visually, not through electrical means.\\n\\n2. By indirect light or Bright lights: Cells are so small that they are not visible to the naked eye under normal lighting conditions. Indirect light or bright lights are not a viable method to observe cells.\\n\\n3. Camera lenses: Camera lenses are used to focus and capture images, not to observe live objects. In order to observe cells, they must be viewed through a microscope which has a lens.\\n\\n4. Colors: Because cells are transparent, they do not have any specific color. They can appear as white or translucent when observed under a microscope.\\n\\n5. Telescope: A telescope is used for observing distant stars, galaxies, and other celestial objects, not for observing cells which are much smaller and earthbound.\\n\\n6. Rays or beams: Cells cannot be observed using rays or beams. Observation of cells is usually done via light microscopy.\\n\\n7. None of the above do not provide the means to observe cells, as they are not the tools or methods designed for cellular observation.\\n\\nThe correct answer (d) A microscope is the tool that allows us to see cells due to its magnifying capabilities making the cells visible to the naked eye effectively.', 'Final Answer': 'c'}</s>\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Define system prompt\n",
+    "system_prompt_RFA = 'Answer the Question and include your Reasoning and the Final Answer in a json like: {\"Reasoning: \"...\", \"Final Answer\": \"x\"} where x is a letter that corresponds to the answer choice which is a letter between a and h.'\n",
+    "\n",
+    "df['assistant_prompt_RFA_gpt3_5'] = df.apply(lambda row: {\"Reasoning\": row[\"gpt3_5_reasoning\"].strip(), \"Final Answer\": row[\"answer_key\"]}, axis=1)\n",
+    "df['assistant_prompt_RFA_mistral'] = df.apply(lambda row: {\"Reasoning\": row[\"mistral_reasoning\"].strip(), \"Final Answer\": row[\"answer_key\"]}, axis=1)\n",
+    "\n",
+    "# Step 1: Create user prompt for both gpt3_5 and mistral\n",
+    "df['user_prompt_RFA'] = df.apply(lambda row: {\n",
+    "    \"content\": system_prompt_RFA + '\\n' + row['user_prompt'],\n",
+    "    \"role\": \"user\"\n",
+    "}, axis=1)\n",
+    "\n",
+    "# Step 2: Create conversation_RFA column using user_prompt_RFA\n",
+    "df['conversation_RFA_gpt3_5'] = df.apply(lambda row: tokenizer.apply_chat_template([\n",
+    "    row['user_prompt_RFA'],  # Use the precomputed user prompt\n",
+    "    {\"content\": row['assistant_prompt_RFA_gpt3_5'], \"role\": \"assistant\"}\n",
+    "], tokenize=False), axis=1)\n",
+    "\n",
+    "df['conversation_RFA_mistral'] = df.apply(lambda row: tokenizer.apply_chat_template([\n",
+    "    row['user_prompt_RFA'],  # Use the precomputed user prompt\n",
+    "    {\"content\": row['assistant_prompt_RFA_mistral'], \"role\": \"assistant\"}\n",
+    "], tokenize=False), axis=1)\n",
+    "\n",
+    "df['user_prompt_RFA'] = df['user_prompt_RFA'].apply(lambda row: tokenizer.apply_chat_template([row], tokenize=False))\n",
+    "\n",
+    "df.drop(['assistant_prompt_RFA_gpt3_5', 'assistant_prompt_RFA_mistral'], inplace=True, axis=1)\n",
+    "\n",
+    "# Example output\n",
+    "gpt3_5_example = df['conversation_RFA_gpt3_5'].iloc[0]\n",
+    "mistral_example = df['conversation_RFA_mistral'].iloc[0]\n",
+    "\n",
+    "print(gpt3_5_example)\n",
+    "print('\\n\\n')\n",
+    "print(mistral_example)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a1150ae9-cc22-44c6-8a36-0e3d7c16111b",
+   "metadata": {},
+   "source": [
+    "### Final Answer Reasoning Structured Generation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "c0234242-7389-4c28-ae85-78a82498d0ac",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<s>[INST] Answer the Question and include your Final Answer and the Reasoning in a json like: {\"Final Answer\": \"x\", \"Reasoning: \"...\"} where x is a letter that corresponds to the answer choice which is a letter between a and h.\n",
+      "Question: What is satellite technology used for predicting?\n",
+      "Answer Choices: (a) Seconds and minutes (b) The strength and magnitude of an earthquake (c) What it's like outside each day (d) 70-75 degrees fahrenheit (e) Rapid changes occur (f) Dead-ends and false starts. (g) Snow, ice, and rock (h) Around 5 to 27 degrees celsius[/INST] {'Final Answer': 'c', 'Reasoning': \"a) Seconds and minutes: This option is incorrect because satellite technology is not used for predicting time intervals. Satellite technology is used for various purposes such as communication, navigation, and weather forecasting, but it is not used for predicting time intervals.\\n\\nb) The strength and magnitude of an earthquake: This option is incorrect because satellite technology is not used for predicting earthquakes. Earthquake prediction is a complex process that involves seismology and other scientific methods, but satellite technology is not one of them.\\n\\nd) 70-75 degrees Fahrenheit: This option is incorrect because satellite technology is not used for predicting specific temperature ranges. While satellite technology can provide temperature data, it is not used for predicting specific temperature ranges.\\n\\ne) Rapid changes occur: This option is too vague and does not provide enough information to determine whether it is correct or not. Satellite technology can be used to monitor changes in various environmental factors, but it is not used specifically for predicting rapid changes.\\n\\nf) Dead-ends and false starts: This option is incorrect because it is not related to satellite technology or any type of prediction.\\n\\ng) Snow, ice, and rock: This option is incorrect because it is too specific and does not cover the broad range of predictions that satellite technology can be used for. While satellite technology can be used to monitor snow, ice, and rock formations, it is not used exclusively for this purpose.\\n\\nh) Around 5 to 27 degrees Celsius: This option is incorrect because it is too specific and does not cover the broad range of temperature predictions that satellite technology can be used for. While satellite technology can provide temperature data, it is not used exclusively for predicting temperatures within a specific range.\\n\\nTherefore, the correct answer is c) what it's like outside each day, as satellite technology is commonly used for weather forecasting and predicting daily weather conditions.\"}</s>\n",
+      "\n",
+      "\n",
+      "\n",
+      "<s>[INST] Answer the Question and include your Final Answer and the Reasoning in a json like: {\"Final Answer\": \"x\", \"Reasoning: \"...\"} where x is a letter that corresponds to the answer choice which is a letter between a and h.\n",
+      "Question: What is satellite technology used for predicting?\n",
+      "Answer Choices: (a) Seconds and minutes (b) The strength and magnitude of an earthquake (c) What it's like outside each day (d) 70-75 degrees fahrenheit (e) Rapid changes occur (f) Dead-ends and false starts. (g) Snow, ice, and rock (h) Around 5 to 27 degrees celsius[/INST] {'Final Answer': 'c', 'Reasoning': 'Incorrect answers and explanations:\\n\\n1. Electrical energy: Cells are not visible with electrical energy. Cells are physical structures and need to be observed visually, not through electrical means.\\n\\n2. By indirect light or Bright lights: Cells are so small that they are not visible to the naked eye under normal lighting conditions. Indirect light or bright lights are not a viable method to observe cells.\\n\\n3. Camera lenses: Camera lenses are used to focus and capture images, not to observe live objects. In order to observe cells, they must be viewed through a microscope which has a lens.\\n\\n4. Colors: Because cells are transparent, they do not have any specific color. They can appear as white or translucent when observed under a microscope.\\n\\n5. Telescope: A telescope is used for observing distant stars, galaxies, and other celestial objects, not for observing cells which are much smaller and earthbound.\\n\\n6. Rays or beams: Cells cannot be observed using rays or beams. Observation of cells is usually done via light microscopy.\\n\\n7. None of the above do not provide the means to observe cells, as they are not the tools or methods designed for cellular observation.\\n\\nThe correct answer (d) A microscope is the tool that allows us to see cells due to its magnifying capabilities making the cells visible to the naked eye effectively.'}</s>\n"
+     ]
+    }
+   ],
+   "source": [
+    "system_prompt_FAR = 'Answer the Question and include your Final Answer and the Reasoning in a json like: {\"Final Answer\": \"x\", \"Reasoning: \"...\"} where x is a letter that corresponds to the answer choice which is a letter between a and h.'\n",
+    "\n",
+    "df['assistant_prompt_FAR_gpt3_5'] = df.apply(lambda row: {\"Final Answer\": row[\"answer_key\"], \"Reasoning\": row[\"gpt3_5_reasoning\"].strip()}, axis=1)\n",
+    "df['assistant_prompt_FAR_mistral'] = df.apply(lambda row: {\"Final Answer\": row[\"answer_key\"], \"Reasoning\": row[\"mistral_reasoning\"].strip()}, axis=1)\n",
+    "\n",
+    "# Step 1: Create user_prompt_FAR column\n",
+    "df['user_prompt_FAR'] = df.apply(lambda row: {\n",
+    "    \"content\": system_prompt_FAR + '\\n' + row['user_prompt'],\n",
+    "    \"role\": \"user\"\n",
+    "}, axis=1)\n",
+    "\n",
+    "# Step 2: Create conversation_FAR column using user_prompt_FAR\n",
+    "df['conversation_FAR_gpt3_5'] = df.apply(lambda row: tokenizer.apply_chat_template([\n",
+    "    row['user_prompt_FAR'],  # Use the precomputed user prompt\n",
+    "    {\"content\": row['assistant_prompt_FAR_gpt3_5'], \"role\": \"assistant\"}\n",
+    "], tokenize=False), axis=1)\n",
+    "\n",
+    "df['conversation_FAR_mistral'] = df.apply(lambda row: tokenizer.apply_chat_template([\n",
+    "    row['user_prompt_FAR'],  # Use the precomputed user prompt\n",
+    "    {\"content\": row['assistant_prompt_FAR_mistral'], \"role\": \"assistant\"}\n",
+    "], tokenize=False), axis=1)\n",
+    "\n",
+    "df['user_prompt_FAR'] = df['user_prompt_FAR'].apply(lambda row: tokenizer.apply_chat_template([row], tokenize=False))\n",
+    "\n",
+    "df.drop(['assistant_prompt_FAR_gpt3_5', 'assistant_prompt_FAR_mistral'], inplace=True, axis=1)\n",
+    "\n",
+    "# Example output\n",
+    "gpt3_5_example = df['conversation_FAR_gpt3_5'].iloc[0]\n",
+    "mistral_example = df['conversation_FAR_mistral'].iloc[0]\n",
+    "\n",
+    "print(gpt3_5_example)\n",
+    "print('\\n\\n')\n",
+    "print(mistral_example)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2c9f0619-1467-4b4e-ad69-b5ca2c4e58a7",
+   "metadata": {},
+   "source": [
+    "### Final Answer Structured Generation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "64dd3601-a40e-478d-97a5-5728005e5787",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<s>[INST] Answer the Question and include your Final Answer in a json like: {\"Final Answer\": \"x\"} where x is a letter that corresponds to the answer choice which is a letter between a and h.\n",
+      "Question: What is satellite technology used for predicting?\n",
+      "Answer Choices: (a) Seconds and minutes (b) The strength and magnitude of an earthquake (c) What it's like outside each day (d) 70-75 degrees fahrenheit (e) Rapid changes occur (f) Dead-ends and false starts. (g) Snow, ice, and rock (h) Around 5 to 27 degrees celsius[/INST] {'Final Answer': 'c'}</s>\n"
+     ]
+    }
+   ],
+   "source": [
+    "system_prompt_FA = 'Answer the Question and include your Final Answer in a json like: {\"Final Answer\": \"x\"} where x is a letter that corresponds to the answer choice which is a letter between a and h.'\n",
+    "df['assistant_prompt_FA'] = df.apply(lambda row: {\"Final Answer\": row[\"answer_key\"]}, axis=1)\n",
+    "\n",
+    "# Step 1: Create user_prompt_FA column\n",
+    "df['user_prompt_FA'] = df.apply(lambda row: {\n",
+    "    \"content\": system_prompt_FA + '\\n' + row['user_prompt'],\n",
+    "    \"role\": \"user\"\n",
+    "}, axis=1)\n",
+    "\n",
+    "# Step 2: Create conversation_FA_R column using user_prompt_FA\n",
+    "df['conversation_FA'] = df.apply(lambda row: tokenizer.apply_chat_template([\n",
+    "    row['user_prompt_FA'],  # Use the precomputed user prompt\n",
+    "    {\"content\": row['assistant_prompt_FA'], \"role\": \"assistant\"}\n",
+    "    # {\"content\": json.dumps(row['assistant_prompt_FA']), \"role\": \"assistant\"}\n",
+    "], tokenize=False), axis=1)\n",
+    "\n",
+    "\n",
+    "df['user_prompt_FA'] = df['user_prompt_FA'].apply(lambda row: tokenizer.apply_chat_template([row], tokenize=False))\n",
+    "\n",
+    "df.drop(['assistant_prompt_FA'], inplace=True, axis=1)\n",
+    "\n",
+    "\n",
+    "# Example output\n",
+    "example = df['conversation_FA'].iloc[0]\n",
+    "\n",
+    "print(example)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ad51b99b-cf67-43d8-833c-5ec57d9d4613",
+   "metadata": {},
+   "source": [
+    "### Cleanup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "69a687d5-35ab-4abb-8bb8-f975fa7be3f7",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['formatted_question', 'combined_fact', 'answer_key', 'topic',\n",
+       "       'gpt3_5_reasoning', 'question_text', 'answer_choices',\n",
+       "       'mistral_reasoning', 'user_prompt', 'user_prompt_RFA',\n",
+       "       'conversation_RFA_gpt3_5', 'conversation_RFA_mistral',\n",
+       "       'user_prompt_FAR', 'conversation_FAR_gpt3_5',\n",
+       "       'conversation_FAR_mistral', 'user_prompt_FA', 'conversation_FA'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "6ec3b6b5-a359-4da8-98c8-4dae75b8a2d4",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "df = df[['topic', 'question_text', 'answer_key', 'gpt3_5_reasoning', 'mistral_reasoning', 'answer_choices', 'user_prompt', \n",
+    "         'user_prompt_RFA', 'conversation_RFA_gpt3_5', 'conversation_RFA_mistral',\n",
+    "         'user_prompt_FAR', 'conversation_FAR_gpt3_5', 'conversation_FAR_mistral',\n",
+    "         'user_prompt_FA', 'conversation_FA']]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "f9f3a5a8-e4ea-4ed9-9fc0-fdd1a19b1c92",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>topic</th>\n",
+       "      <th>question_text</th>\n",
+       "      <th>answer_key</th>\n",
+       "      <th>gpt3_5_reasoning</th>\n",
+       "      <th>mistral_reasoning</th>\n",
+       "      <th>answer_choices</th>\n",
+       "      <th>user_prompt</th>\n",
+       "      <th>user_prompt_RFA</th>\n",
+       "      <th>conversation_RFA_gpt3_5</th>\n",
+       "      <th>conversation_RFA_mistral</th>\n",
+       "      <th>user_prompt_FAR</th>\n",
+       "      <th>conversation_FAR_gpt3_5</th>\n",
+       "      <th>conversation_FAR_mistral</th>\n",
+       "      <th>user_prompt_FA</th>\n",
+       "      <th>conversation_FA</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Technology</td>\n",
+       "      <td>What is satellite technology used for predicting?</td>\n",
+       "      <td>c</td>\n",
+       "      <td>a) Seconds and minutes: This option is incorre...</td>\n",
+       "      <td>Incorrect answers and explanations:\\n\\n1. Elec...</td>\n",
+       "      <td>(a) Seconds and minutes (b) The strength and m...</td>\n",
+       "      <td>Question: What is satellite technology used fo...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Food science</td>\n",
+       "      <td>What does irradiating food do?</td>\n",
+       "      <td>c</td>\n",
+       "      <td>(a) Relieve pain: This option is not correct b...</td>\n",
+       "      <td>Sure, let's examine each answer and justify wh...</td>\n",
+       "      <td>(a) Relieve pain (b) Enhance food's nutrients ...</td>\n",
+       "      <td>Question: What does irradiating food do?\\nAnsw...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Biology</td>\n",
+       "      <td>What protects a mammal's skin?</td>\n",
+       "      <td>a</td>\n",
+       "      <td>b) Exfoliation: Exfoliation is the process of ...</td>\n",
+       "      <td>Sure, let's go through each of the provided an...</td>\n",
+       "      <td>(a) Fiber follicles (b) Exfoliation (c) Resist...</td>\n",
+       "      <td>Question: What protects a mammal's skin?\\nAnsw...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Biology</td>\n",
+       "      <td>What do earthworms do when a segment breaks off?</td>\n",
+       "      <td>b</td>\n",
+       "      <td>a) Dies: This option is not correct because ea...</td>\n",
+       "      <td>1. Reading the question carefully, we can see ...</td>\n",
+       "      <td>(a) Dies (b) Regrows it (c) Reproduces (d) Sed...</td>\n",
+       "      <td>Question: What do earthworms do when a segment...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Electricity</td>\n",
+       "      <td>Lightning can be bad for what?</td>\n",
+       "      <td>a</td>\n",
+       "      <td>b) Rainstorms: Lightning is actually a natural...</td>\n",
+       "      <td>1. Food: While essential for the growth and he...</td>\n",
+       "      <td>(a) The environment (b) Rainstorms (c) Destruc...</td>\n",
+       "      <td>Question: Lightning can be bad for what?\\nAnsw...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8408</th>\n",
+       "      <td>Biology</td>\n",
+       "      <td>Organisms that can cause infection do what?</td>\n",
+       "      <td>g</td>\n",
+       "      <td>a) Bandaging open sores is not the correct ans...</td>\n",
+       "      <td>1. Read the question and options carefully: Th...</td>\n",
+       "      <td>(a) Bandage open sores (b) Keep flesh clean (c...</td>\n",
+       "      <td>Question: Organisms that can cause infection d...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8409</th>\n",
+       "      <td>Biology</td>\n",
+       "      <td>Fungi are living things that cannot make their...</td>\n",
+       "      <td>a</td>\n",
+       "      <td>b) Fungi are living things that can make their...</td>\n",
+       "      <td>1. Read the question and options carefully: Th...</td>\n",
+       "      <td>(a) Food (b) Cells (c) Energy (d) Fruits (e) H...</td>\n",
+       "      <td>Question: Fungi are living things that cannot ...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8410</th>\n",
+       "      <td>Biology</td>\n",
+       "      <td>An overheated body can use water for:?</td>\n",
+       "      <td>g</td>\n",
+       "      <td>a) Metabolic reaction: This option is incorrec...</td>\n",
+       "      <td>1. Read the question and options carefully: Th...</td>\n",
+       "      <td>(a) Metabolic reaction (b) Dehydrating (c) Rai...</td>\n",
+       "      <td>Question: An overheated body can use water for...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8411</th>\n",
+       "      <td>Biology</td>\n",
+       "      <td>What is essential for cellular respiration for...</td>\n",
+       "      <td>f</td>\n",
+       "      <td>a) Electrons are involved in cellular respirat...</td>\n",
+       "      <td>1. First, let's read the question and options ...</td>\n",
+       "      <td>(a) Electron (b) Glucose (c) Energy (d) Energy...</td>\n",
+       "      <td>Question: What is essential for cellular respi...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8412</th>\n",
+       "      <td>Biology</td>\n",
+       "      <td>What helps insulate and protect the body?</td>\n",
+       "      <td>b</td>\n",
+       "      <td>a) H2O: Water is essential for life, but it do...</td>\n",
+       "      <td>1. Read the question and options carefully: Th...</td>\n",
+       "      <td>(a) H2o (b) Living cells in follicles (c) Laye...</td>\n",
+       "      <td>Question: What helps insulate and protect the ...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>8413 rows × 15 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "             topic                                      question_text  \\\n",
+       "0       Technology  What is satellite technology used for predicting?   \n",
+       "1     Food science                     What does irradiating food do?   \n",
+       "2          Biology                     What protects a mammal's skin?   \n",
+       "3          Biology   What do earthworms do when a segment breaks off?   \n",
+       "4      Electricity                     Lightning can be bad for what?   \n",
+       "...            ...                                                ...   \n",
+       "8408       Biology        Organisms that can cause infection do what?   \n",
+       "8409       Biology  Fungi are living things that cannot make their...   \n",
+       "8410       Biology             An overheated body can use water for:?   \n",
+       "8411       Biology  What is essential for cellular respiration for...   \n",
+       "8412       Biology          What helps insulate and protect the body?   \n",
+       "\n",
+       "     answer_key                                   gpt3_5_reasoning  \\\n",
+       "0             c  a) Seconds and minutes: This option is incorre...   \n",
+       "1             c  (a) Relieve pain: This option is not correct b...   \n",
+       "2             a  b) Exfoliation: Exfoliation is the process of ...   \n",
+       "3             b  a) Dies: This option is not correct because ea...   \n",
+       "4             a  b) Rainstorms: Lightning is actually a natural...   \n",
+       "...         ...                                                ...   \n",
+       "8408          g  a) Bandaging open sores is not the correct ans...   \n",
+       "8409          a  b) Fungi are living things that can make their...   \n",
+       "8410          g  a) Metabolic reaction: This option is incorrec...   \n",
+       "8411          f  a) Electrons are involved in cellular respirat...   \n",
+       "8412          b  a) H2O: Water is essential for life, but it do...   \n",
+       "\n",
+       "                                      mistral_reasoning  \\\n",
+       "0     Incorrect answers and explanations:\\n\\n1. Elec...   \n",
+       "1     Sure, let's examine each answer and justify wh...   \n",
+       "2     Sure, let's go through each of the provided an...   \n",
+       "3     1. Reading the question carefully, we can see ...   \n",
+       "4     1. Food: While essential for the growth and he...   \n",
+       "...                                                 ...   \n",
+       "8408  1. Read the question and options carefully: Th...   \n",
+       "8409  1. Read the question and options carefully: Th...   \n",
+       "8410  1. Read the question and options carefully: Th...   \n",
+       "8411  1. First, let's read the question and options ...   \n",
+       "8412  1. Read the question and options carefully: Th...   \n",
+       "\n",
+       "                                         answer_choices  \\\n",
+       "0     (a) Seconds and minutes (b) The strength and m...   \n",
+       "1     (a) Relieve pain (b) Enhance food's nutrients ...   \n",
+       "2     (a) Fiber follicles (b) Exfoliation (c) Resist...   \n",
+       "3     (a) Dies (b) Regrows it (c) Reproduces (d) Sed...   \n",
+       "4     (a) The environment (b) Rainstorms (c) Destruc...   \n",
+       "...                                                 ...   \n",
+       "8408  (a) Bandage open sores (b) Keep flesh clean (c...   \n",
+       "8409  (a) Food (b) Cells (c) Energy (d) Fruits (e) H...   \n",
+       "8410  (a) Metabolic reaction (b) Dehydrating (c) Rai...   \n",
+       "8411  (a) Electron (b) Glucose (c) Energy (d) Energy...   \n",
+       "8412  (a) H2o (b) Living cells in follicles (c) Laye...   \n",
+       "\n",
+       "                                            user_prompt  \\\n",
+       "0     Question: What is satellite technology used fo...   \n",
+       "1     Question: What does irradiating food do?\\nAnsw...   \n",
+       "2     Question: What protects a mammal's skin?\\nAnsw...   \n",
+       "3     Question: What do earthworms do when a segment...   \n",
+       "4     Question: Lightning can be bad for what?\\nAnsw...   \n",
+       "...                                                 ...   \n",
+       "8408  Question: Organisms that can cause infection d...   \n",
+       "8409  Question: Fungi are living things that cannot ...   \n",
+       "8410  Question: An overheated body can use water for...   \n",
+       "8411  Question: What is essential for cellular respi...   \n",
+       "8412  Question: What helps insulate and protect the ...   \n",
+       "\n",
+       "                                        user_prompt_RFA  \\\n",
+       "0     <s>[INST] Answer the Question and include your...   \n",
+       "1     <s>[INST] Answer the Question and include your...   \n",
+       "2     <s>[INST] Answer the Question and include your...   \n",
+       "3     <s>[INST] Answer the Question and include your...   \n",
+       "4     <s>[INST] Answer the Question and include your...   \n",
+       "...                                                 ...   \n",
+       "8408  <s>[INST] Answer the Question and include your...   \n",
+       "8409  <s>[INST] Answer the Question and include your...   \n",
+       "8410  <s>[INST] Answer the Question and include your...   \n",
+       "8411  <s>[INST] Answer the Question and include your...   \n",
+       "8412  <s>[INST] Answer the Question and include your...   \n",
+       "\n",
+       "                                conversation_RFA_gpt3_5  \\\n",
+       "0     <s>[INST] Answer the Question and include your...   \n",
+       "1     <s>[INST] Answer the Question and include your...   \n",
+       "2     <s>[INST] Answer the Question and include your...   \n",
+       "3     <s>[INST] Answer the Question and include your...   \n",
+       "4     <s>[INST] Answer the Question and include your...   \n",
+       "...                                                 ...   \n",
+       "8408  <s>[INST] Answer the Question and include your...   \n",
+       "8409  <s>[INST] Answer the Question and include your...   \n",
+       "8410  <s>[INST] Answer the Question and include your...   \n",
+       "8411  <s>[INST] Answer the Question and include your...   \n",
+       "8412  <s>[INST] Answer the Question and include your...   \n",
+       "\n",
+       "                               conversation_RFA_mistral  \\\n",
+       "0     <s>[INST] Answer the Question and include your...   \n",
+       "1     <s>[INST] Answer the Question and include your...   \n",
+       "2     <s>[INST] Answer the Question and include your...   \n",
+       "3     <s>[INST] Answer the Question and include your...   \n",
+       "4     <s>[INST] Answer the Question and include your...   \n",
+       "...                                                 ...   \n",
+       "8408  <s>[INST] Answer the Question and include your...   \n",
+       "8409  <s>[INST] Answer the Question and include your...   \n",
+       "8410  <s>[INST] Answer the Question and include your...   \n",
+       "8411  <s>[INST] Answer the Question and include your...   \n",
+       "8412  <s>[INST] Answer the Question and include your...   \n",
+       "\n",
+       "                                        user_prompt_FAR  \\\n",
+       "0     <s>[INST] Answer the Question and include your...   \n",
+       "1     <s>[INST] Answer the Question and include your...   \n",
+       "2     <s>[INST] Answer the Question and include your...   \n",
+       "3     <s>[INST] Answer the Question and include your...   \n",
+       "4     <s>[INST] Answer the Question and include your...   \n",
+       "...                                                 ...   \n",
+       "8408  <s>[INST] Answer the Question and include your...   \n",
+       "8409  <s>[INST] Answer the Question and include your...   \n",
+       "8410  <s>[INST] Answer the Question and include your...   \n",
+       "8411  <s>[INST] Answer the Question and include your...   \n",
+       "8412  <s>[INST] Answer the Question and include your...   \n",
+       "\n",
+       "                                conversation_FAR_gpt3_5  \\\n",
+       "0     <s>[INST] Answer the Question and include your...   \n",
+       "1     <s>[INST] Answer the Question and include your...   \n",
+       "2     <s>[INST] Answer the Question and include your...   \n",
+       "3     <s>[INST] Answer the Question and include your...   \n",
+       "4     <s>[INST] Answer the Question and include your...   \n",
+       "...                                                 ...   \n",
+       "8408  <s>[INST] Answer the Question and include your...   \n",
+       "8409  <s>[INST] Answer the Question and include your...   \n",
+       "8410  <s>[INST] Answer the Question and include your...   \n",
+       "8411  <s>[INST] Answer the Question and include your...   \n",
+       "8412  <s>[INST] Answer the Question and include your...   \n",
+       "\n",
+       "                               conversation_FAR_mistral  \\\n",
+       "0     <s>[INST] Answer the Question and include your...   \n",
+       "1     <s>[INST] Answer the Question and include your...   \n",
+       "2     <s>[INST] Answer the Question and include your...   \n",
+       "3     <s>[INST] Answer the Question and include your...   \n",
+       "4     <s>[INST] Answer the Question and include your...   \n",
+       "...                                                 ...   \n",
+       "8408  <s>[INST] Answer the Question and include your...   \n",
+       "8409  <s>[INST] Answer the Question and include your...   \n",
+       "8410  <s>[INST] Answer the Question and include your...   \n",
+       "8411  <s>[INST] Answer the Question and include your...   \n",
+       "8412  <s>[INST] Answer the Question and include your...   \n",
+       "\n",
+       "                                         user_prompt_FA  \\\n",
+       "0     <s>[INST] Answer the Question and include your...   \n",
+       "1     <s>[INST] Answer the Question and include your...   \n",
+       "2     <s>[INST] Answer the Question and include your...   \n",
+       "3     <s>[INST] Answer the Question and include your...   \n",
+       "4     <s>[INST] Answer the Question and include your...   \n",
+       "...                                                 ...   \n",
+       "8408  <s>[INST] Answer the Question and include your...   \n",
+       "8409  <s>[INST] Answer the Question and include your...   \n",
+       "8410  <s>[INST] Answer the Question and include your...   \n",
+       "8411  <s>[INST] Answer the Question and include your...   \n",
+       "8412  <s>[INST] Answer the Question and include your...   \n",
+       "\n",
+       "                                        conversation_FA  \n",
+       "0     <s>[INST] Answer the Question and include your...  \n",
+       "1     <s>[INST] Answer the Question and include your...  \n",
+       "2     <s>[INST] Answer the Question and include your...  \n",
+       "3     <s>[INST] Answer the Question and include your...  \n",
+       "4     <s>[INST] Answer the Question and include your...  \n",
+       "...                                                 ...  \n",
+       "8408  <s>[INST] Answer the Question and include your...  \n",
+       "8409  <s>[INST] Answer the Question and include your...  \n",
+       "8410  <s>[INST] Answer the Question and include your...  \n",
+       "8411  <s>[INST] Answer the Question and include your...  \n",
+       "8412  <s>[INST] Answer the Question and include your...  \n",
+       "\n",
+       "[8413 rows x 15 columns]"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9fe52e77-0985-49d4-964e-7a17372d7007",
+   "metadata": {},
+   "source": [
+    "## Explore Prompts\n",
+    "Gradio can be really useful for quick inline apps. Here I want to make sure everything is as I expect.\n",
+    "\n",
+    "While the above print statements helped me see the format, the gradio app helps me explore a large volume of output easily. \n",
+    "\n",
+    "Note: Its tricky as I cant easily render newlines in strings. So be careful!"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "a50d9d6c-18e6-476d-9a40-ed7a3f699477",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "* Running on local URL:  http://127.0.0.1:7861\n",
+      "\n",
+      "To create a public link, set `share=True` in `launch()`.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div><iframe src=\"http://127.0.0.1:7861/\" width=\"100%\" height=\"840\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": []
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import json\n",
+    "import gradio as gr\n",
+    "\n",
+    "# Gradio app to browse prompts with left and right buttons\n",
+    "# index = 0\n",
+    "\n",
+    "# Functions to handle prompts\n",
+    "def get_prompt(index, prompt_type):\n",
+    "    return df.iloc[index][prompt_type]\n",
+    "\n",
+    "def next_prompt(index, prompt_type):\n",
+    "    if index < len(df) - 1:\n",
+    "        index += 1\n",
+    "    return index, get_prompt(index, prompt_type)\n",
+    "\n",
+    "def previous_prompt(index, prompt_type):\n",
+    "    if index > 0:\n",
+    "        index -= 1\n",
+    "    return index, get_prompt(index, prompt_type)\n",
+    "\n",
+    "# Gradio App\n",
+    "with gr.Blocks() as demo:\n",
+    "    gr.Markdown(\"# Prompt Browser\")\n",
+    "    with gr.Row():\n",
+    "        prompt_type_dropdown = gr.Dropdown(\n",
+    "            choices=['conversation_RFA_gpt3_5', 'conversation_RFA_mistral', 'conversation_FAR_gpt3_5', 'conversation_FAR_mistral', 'conversation_FA_gpt3_5', 'conversation_FA_mistral'],\n",
+    "            value='conversation_RFA_gpt3_5',\n",
+    "            label=\"Select Prompt Type\"\n",
+    "        )\n",
+    "        index_display = gr.Textbox(\"0\", label=\"Index\", interactive=False)\n",
+    "\n",
+    "    prompt_display = gr.Textbox(value=df.iloc[0]['conversation_RFA_gpt3_5'], label=\"Prompt\")\n",
+    "    \n",
+    "    with gr.Row():\n",
+    "        prev_button = gr.Button(\"⬅️ Previous\")\n",
+    "        next_button = gr.Button(\"Next ➡️\")\n",
+    "    \n",
+    "    # State to hold the current index\n",
+    "    index_state = gr.State(value=0)\n",
+    "\n",
+    "    # Button click events\n",
+    "    prev_button.click(\n",
+    "        fn=previous_prompt,\n",
+    "        inputs=[index_state, prompt_type_dropdown],\n",
+    "        outputs=[index_state, prompt_display]\n",
+    "    )\n",
+    "    next_button.click(\n",
+    "        fn=next_prompt,\n",
+    "        inputs=[index_state, prompt_type_dropdown],\n",
+    "        outputs=[index_state, prompt_display]\n",
+    "    )\n",
+    "\n",
+    "    # Dropdown change event\n",
+    "    prompt_type_dropdown.change(\n",
+    "        fn=lambda index, prompt_type: get_prompt(index, prompt_type),\n",
+    "        inputs=[index_state, prompt_type_dropdown],\n",
+    "        outputs=prompt_display\n",
+    "    )\n",
+    "\n",
+    "    # Update index display\n",
+    "    index_state.change(\n",
+    "        fn=lambda index: str(index),\n",
+    "        inputs=index_state,\n",
+    "        outputs=index_display\n",
+    "    )\n",
+    "\n",
+    "# Launch the app\n",
+    "demo.launch(height=840)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3838daf0-8a3a-4513-ad2b-3589d60dfa3d",
+   "metadata": {},
+   "source": [
+    "## Push Dataset to the Hub\n",
+    "Its useful to get a train, test split, then we convert to `Dataset` and push to the hub. We also want to stratify on `'topic'`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "25f62e9b-09f8-4912-94fd-0ded680614b2",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from datasets import Dataset, DatasetDict\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "\n",
+    "# First split to create train and remaining (val + test)\n",
+    "train_df, test_df = train_test_split(df, test_size=0.2, stratify=df['topic'], random_state=42)\n",
+    "\n",
+    "# Reset index to avoid index column in the Hugging Face Dataset\n",
+    "train_df.reset_index(drop=True, inplace=True)\n",
+    "test_df.reset_index(drop=True, inplace=True)\n",
+    "\n",
+    "# Convert each DataFrame to a Dataset object\n",
+    "train_dataset = Dataset.from_pandas(train_df)\n",
+    "test_dataset = Dataset.from_pandas(test_df)\n",
+    "\n",
+    "# Create a DatasetDict with the train, validation, and test datasets\n",
+    "dataset_dict = DatasetDict({\n",
+    "    'train': train_dataset,\n",
+    "    'test': test_dataset\n",
+    "})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "18a206c5-0e40-46b3-8dfb-20000789b6b5",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0e40274a780c4da0868362339d48e6de",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d8ed56081baa4b4a98af22b3d904a231",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Creating parquet from Arrow format:   0%|          | 0/7 [00:00<?, ?ba/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c6b2c92b5aaa4943ab8b086f58455440",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f4a05ea3069c4f60bca73b488f8221d5",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Creating parquet from Arrow format:   0%|          | 0/2 [00:00<?, ?ba/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "CommitInfo(commit_url='https://huggingface.co/datasets/derek-thomas/labeled-multiple-choice-explained-mistral-tokenized/commit/f96a8487961dcfe6077df67b5351c041a4523eb1', commit_message='Upload dataset', commit_description='', oid='f96a8487961dcfe6077df67b5351c041a4523eb1', pr_url=None, repo_url=RepoUrl('https://huggingface.co/datasets/derek-thomas/labeled-multiple-choice-explained-mistral-tokenized', endpoint='https://huggingface.co', repo_type='dataset', repo_id='derek-thomas/labeled-multiple-choice-explained-mistral-tokenized'), pr_revision=None, pr_num=None)"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Push the dataset to the Hugging Face Hub\n",
+    "dataset_dict.push_to_hub(OUTPUT_DATASET)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "14c279ea-4f3b-40bd-9795-edd5ea3694f7",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

02-poe-token-count-exploration.ipynb

@@ -0,0 +1,435 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "b3fc8862-0c2b-45f3-badf-e591c7b8f891",
+   "metadata": {},
+   "source": [
+    "# Token Count Exploration\n",
+    "It would be really useful for deployment to know our input/output expectations. We know that our output is quite verbose relative to the input since the explanations are long. With a model like `mistralai/Mistral-7B-Instruct-v0.3` Id expect that our real output with explanations will be shorter. Thats perfect since our training data will give us a reliable upper bound, which is great to prevent truncation.\n",
+    "\n",
+    "Lets figure out how to split input and output tokens, and then we can build a histogram."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3a501f2f-ba98-4c0f-aa30-f4768bd80dcb",
+   "metadata": {},
+   "source": [
+    "## Config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "5d0bd22f-293e-4c15-9dfe-8070553f42b5",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "INPUT_DATASET = 'derek-thomas/labeled-multiple-choice-explained-mistral-tokenized'\n",
+    "BASE_MODEL = 'mistralai/Mistral-7B-Instruct-v0.3'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c1c3b00c-17bf-4b00-9ee7-d10c598c53e9",
+   "metadata": {},
+   "source": [
+    "## Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "af2330f3-403c-401c-8028-46ae4971546e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d675da3076694064ba0c69ed97f938f8",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from huggingface_hub import login, get_token\n",
+    "login()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "a9e2d29c-1f8e-4a70-839f-f61ae396d6f6",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "dd06a12730fa4af1b863273c333c6a4c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "README.md:   0%|          | 0.00/1.18k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0ab7651927dc407f83c819dffc2c6cf1",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00000-of-00001.parquet:   0%|          | 0.00/40.5M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "93eec90b188d4b4b862cba87fbc65f26",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "test-00000-of-00001.parquet:   0%|          | 0.00/10.1M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "21a9ed213b2c4da49f6171206102499d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Generating train split:   0%|          | 0/6730 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "52a994708c2a466f8ed72a2fd881aa77",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Generating test split:   0%|          | 0/1683 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from transformers import AutoTokenizer\n",
+    "from datasets import load_dataset\n",
+    "\n",
+    "tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, token=get_token())\n",
+    "\n",
+    "dataset = load_dataset(INPUT_DATASET, split='train')\n",
+    "df = dataset.to_pandas()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f3a644bf-b532-4f30-bd6b-c637dfde5fb2",
+   "metadata": {},
+   "source": [
+    "## Exploration"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "bf5b3e0c-2b7f-42f3-852d-7039c530ed86",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'<s>[INST] Answer the Question and include your Reasoning and the Final Answer in a json like: {\"Reasoning: \"...\", \"Final Answer\": \"x\"} where x is a letter that corresponds to the answer choice which is a letter between a and h.\\nQuestion: What can genetic material have?\\nAnswer Choices: (a) Resistance (b) Mutations (c) Clorophyll (d) Nucleotide (e) Symmetry (f) Allow growth (g) Contamination (h) Warmth[/INST] {\\'Reasoning\\': \\'a) Resistance: Genetic material can carry genes that provide resistance to certain diseases or environmental factors, but this is not a characteristic of genetic material itself. Therefore, this option is incorrect.\\\\n\\\\nc) Chlorophyll: Chlorophyll is a pigment found in plants that is responsible for photosynthesis. It is not a characteristic of genetic material. Therefore, this option is incorrect.\\\\n\\\\nd) Nucleotide: Nucleotides are the building blocks of DNA and RNA, which are types of genetic material. However, this option is too broad and does not fully answer the question. Therefore, this option is incorrect.\\\\n\\\\ne) Symmetry: Symmetry is a characteristic of physical objects and organisms, but it is not a characteristic of genetic material. Therefore, this option is incorrect.\\\\n\\\\nf) Allow growth: Genetic material provides the instructions for the growth and development of organisms, but it is not a characteristic of genetic material itself. Therefore, this option is incorrect.\\\\n\\\\ng) Contamination: Contamination is the presence of unwanted substances or impurities, and it is not a characteristic of genetic material. Therefore, this option is incorrect.\\\\n\\\\nh) Warmth: Warmth is a physical property of objects and is not related to genetic material. Therefore, this option is incorrect.\\\\n\\\\nIn conclusion, the only option that correctly describes a characteristic of genetic material is b) mutations. Genetic material can have mutations, which are changes in the DNA sequence that can lead to genetic variation and evolution.\\', \\'Final Answer\\': \\'b\\'}</s>'"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df['conversation_RFA_sg_gpt3_5'].iloc[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "0dc985d7-32e3-413f-8640-55829da19838",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[4]"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokenizer.encode('[/INST]', add_special_tokens=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "bc9b3856-7652-483c-8dbc-2b9bdc85f9d7",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[1, 3, 27075, 1040, 23246, 1072, 3792, 1342, 2066, 2180, 1056, 1072, 1040, 10990, 27075, 1065, 1032, 8379, 1505, 29515, 10598, 20569, 1056, 29515, 1113, 1869, 1316, 1113, 18268, 27075, 2032, 1113, 29512, 18163, 1738, 2086, 1117, 1032, 6266, 1137, 17303, 1066, 1040, 5140, 5550, 1458, 1117, 1032, 6266, 2212, 1032, 1072, 1063, 29491, 781, 25762, 29515, 2592, 1309, 20637, 4156, 1274, 29572, 781, 3588, 17749, 26173, 1982, 29515, 1093, 29476, 29499, 2760, 5400, 1093, 29494, 29499, 17737, 1465, 1093, 29485, 29499, 2134, 1039, 3894, 20298, 1093, 29483, 29499, 1186, 2253, 1059, 1090, 1315, 1093, 29474, 29499, 13124, 17409, 1093, 29490, 29499, 26780, 6825, 1093, 29489, 29499, 3767, 26682, 1093, 29484, 29499, 1162, 2553, 1130, 4, 12780, 20569, 1056, 2637, 1232, 29476, 29499, 2760, 5400, 29515, 7010, 11130, 4156, 1309, 7864, 24971, 1137, 3852, 13336, 1066, 3320, 19025, 1210, 13275, 9380, 29493, 1330, 1224, 1117, 1227, 1032, 18613, 1070, 20637, 4156, 4605, 29491, 9237, 29493, 1224, 4319, 1117, 17158, 6691, 29479, 29524, 15538, 29499, 1457, 6406, 3894, 20298, 29515, 1457, 6406, 3894, 20298, 1117, 1032, 19726, 1234, 2187, 1065, 10691, 1137, 1117, 8100, 1122, 9654, 29492, 1216, 22305, 29491, 1429, 1117, 1227, 1032, 18613, 1070, 20637, 4156, 29491, 9237, 29493, 1224, 4319, 1117, 17158, 6691, 29479, 29524, 1060, 29499, 1186, 2253, 1059, 1090, 1315, 29515, 1186, 2253, 1059, 1090, 2694, 1228, 1040, 4435, 10246, 1070, 16775, 1072, 1167, 4152, 29493, 1458, 1228, 5282, 1070, 20637, 4156, 29491, 3761, 29493, 1224, 4319, 1117, 2136, 6609, 1072, 2003, 1227, 6662, 5140, 1040, 3764, 29491, 9237, 29493, 1224, 4319, 1117, 17158, 6691, 29479, 29524, 1253, 29499, 13124, 17409, 29515, 13124, 17409, 1117, 1032, 18613, 1070, 6045, 7465, 1072, 2938, 11589, 29493, 1330, 1146, 1117, 1227, 1032, 18613, 1070, 20637, 4156, 29491, 9237, 29493, 1224, 4319, 1117, 17158, 6691, 29479, 29524, 24412, 29499, 26780, 6825, 29515, 7010, 11130, 4156, 6080, 1040, 12150, 1122, 1040, 6825, 1072, 4867, 1070, 2938, 11589, 29493, 1330, 1146, 1117, 1227, 1032, 18613, 1070, 20637, 4156, 4605, 29491, 9237, 29493, 1224, 4319, 1117, 17158, 6691, 29479, 29524, 1585, 29499, 3767, 26682, 29515, 3767, 26682, 1117, 1040, 7471, 1070, 13460, 8034, 1851, 9500, 1210, 3592, 1092, 1986, 29493, 1072, 1146, 1117, 1227, 1032, 18613, 1070, 20637, 4156, 29491, 9237, 29493, 1224, 4319, 1117, 17158, 6691, 29479, 29524, 25779, 29499, 1162, 2553, 1130, 29515, 1162, 2553, 1130, 1117, 1032, 6045, 4089, 1070, 7465, 1072, 1117, 1227, 5970, 1066, 20637, 4156, 29491, 9237, 29493, 1224, 4319, 1117, 17158, 6691, 29479, 29524, 29479, 1425, 13654, 29493, 1040, 1633, 4319, 1137, 13510, 14734, 1032, 18613, 1070, 20637, 4156, 1117, 1055, 29499, 5316, 1465, 29491, 7010, 11130, 4156, 1309, 1274, 5316, 1465, 29493, 1458, 1228, 5203, 1065, 1040, 16775, 8536, 1137, 1309, 2504, 1066, 20637, 19191, 1072, 10963, 13775, 1232, 18268, 27075, 2637, 1232, 29494, 15259, 2]\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "[4]"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "print(tokenizer.encode(df['conversation_RFA_sg_gpt3_5'].iloc[0], add_special_tokens=False))\n",
+    "tokenizer.encode('[/INST]', add_special_tokens=False)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "677e792e-a85f-448c-ab36-ed0aec84ca8e",
+   "metadata": {},
+   "source": [
+    "Great, we can see that there is a special token `[/INST]` that we will want to split on. We can count the tokens before and including `[/INST]` and that should be our input tokens, and the tokens after will be our output tokens.\n",
+    "\n",
+    "Lets count those for each row in `conversation_RFA` and build a histogram of the results. `conversation_RFA` should be a good max since its just a reshuffle or superset of the other columns."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "3c8cd920-4d58-4b1d-b172-098c35dcdfbf",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from datasets import load_dataset\n",
+    "from transformers import AutoTokenizer\n",
+    "\n",
+    "# Load the dataset and convert it to a DataFrame\n",
+    "dataset = load_dataset(INPUT_DATASET, split='test')\n",
+    "df = dataset.to_pandas()\n",
+    "\n",
+    "df_token_gpt3_5 = df[['conversation_RFA_sg_gpt3_5']].copy()\n",
+    "df_token_gpt3_5['tokens_gpt3_5'] = df['conversation_RFA_sg_gpt3_5'].apply(lambda x: tokenizer.encode(x))\n",
+    "\n",
+    "df_token_mistral = df[['conversation_RFA_sg_mistral']].copy()\n",
+    "df_token_mistral['tokens_mistral'] = df['conversation_RFA_sg_mistral'].apply(lambda x: tokenizer.encode(x))\n",
+    "\n",
+    "def split_and_measure(lst):\n",
+    "    if 4 in lst:\n",
+    "        index_of_4 = lst.index(4)\n",
+    "        length_before = index_of_4 + 1  # Including 4\n",
+    "        length_after = len(lst) - length_before\n",
+    "        return length_before, length_after\n",
+    "    else:\n",
+    "        return None, len(lst)  # If 4 is not present\n",
+    "\n",
+    "df_token_gpt3_5[['input_tokens', 'output_tokens']] = df_token_gpt3_5['tokens_gpt3_5'].apply(split_and_measure).apply(pd.Series)\n",
+    "df_token_mistral[['input_tokens', 'output_tokens']] = df_token_mistral['tokens_mistral'].apply(split_and_measure).apply(pd.Series)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "9b23b7a3-5448-4b2e-9253-5d1b66ef1e0a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1IAAAIjCAYAAAAJLyrXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABK9klEQVR4nO3de3zP9f//8ft7RzvYZuxAZshxDCEslJBhRBRpH0Y+9Y0ROXzK71OOfUwKEeLbp4w+DqWvQzmFYRVTyFhIaJmyg5LNyDbb6/dHF+9P74b2mm3v4Xa9XF6Xi9fz+Xy9Xo/X+/K60L3n62AxDMMQAAAAAKDIHOxdAAAAAADcbghSAAAAAGASQQoAAAAATCJIAQAAAIBJBCkAAAAAMIkgBQAAAAAmEaQAAAAAwCSCFAAAAACYRJACAAAAAJMIUgCAEmWxWDRixAh7lwEAQKkiSAEAZLFYirTs2rXL3qWaUlBQoGXLlql169by9fVVxYoVVa9ePQ0aNEh79+61d3kAgNuYk70LAADY3/vvv2+zvmzZMm3btq1Qe8OGDcuyrFv2/PPPa8GCBerVq5ciIyPl5OSk48ePa/Pmzapdu7batGlj7xIBALcpi2EYhr2LAACULyNGjNCCBQtUnH8iLBaLoqOjNX/+/FKorOjS09NVtWpV/f3vf9f//u//2vQZhqFz587J39/fTtXZx5UrV+Ti4iIHB25IAYBbxd+kAIAiuXTpksaOHaugoCC5urqqfv36euONN4oUtl599VU5ODjorbfesrZt3rxZ7du3l4eHhypWrKiIiAgdOXLEZrvBgwfL09NTP/30k3r37i1PT0/5+flp3Lhxys/Pv+kxk5OTZRiG2rZtW6jPYrHYhKjJkyfLYrEUGhcbGyuLxaIffvjB2lazZk316NFDu3btUsuWLeXm5qbQ0FDrbY9r1qxRaGioKlSooBYtWujgwYPXPaeUlBT16NFDnp6euueee7RgwQJJUlJSkjp27CgPDw8FBwdrxYoVNtufP39e48aNU2hoqDw9PeXl5aVu3brp0KFDNuN27doli8WiVatW6eWXX9Y999wjd3d3JSYmymKxaM6cOYXOd8+ePbJYLFq5cuVNf1sAAEEKAFAEhmHo0Ucf1Zw5c9S1a1fNnj1b9evX1/jx4zVmzJibbvvyyy9r4sSJWrx4sUaOHCnp91sJIyIi5Onpqddee02vvPKKjh49qnbt2tmEFknKz89XeHi4KleurDfeeEMPPfSQZs2aVWiW6c+Cg4MlSatXr9bly5eLf/LXcfLkST311FPq2bOnYmJi9Ouvv6pnz55avny5XnjhBf3tb3/TlClTdOrUKfXr108FBQWFzqlbt24KCgrSzJkzVbNmTY0YMUKxsbHq2rWrWrZsqddee00VK1bUoEGDlJycbN32+++/17p169SjRw/Nnj1b48ePV1JSkh566CGdPXu2UK3Tpk3Txo0bNW7cOE2fPl0NGjRQ27ZttXz58kJjly9frooVK6pXr14l+nsBwB3JAADgT6Kjo40//hOxbt06Q5Lx6quv2ox7/PHHDYvFYpw8edLaJsmIjo42DMMwxo4dazg4OBixsbHW/osXLxo+Pj7GM888Y7OvtLQ0w9vb26Y9KirKkGRMnTrVZux9991ntGjR4i/PY9CgQYYko1KlSsZjjz1mvPHGG8axY8cKjZs0aZJxvX8SlyxZYkgykpOTrW3BwcGGJGPPnj3Wtk8//dSQZLi5uRmnT5+2ti9evNiQZOzcubPQOU2fPt3a9uuvvxpubm6GxWIxVq1aZW3/9ttvDUnGpEmTrG1Xrlwx8vPzbepMTk42XF1dbX6nnTt3GpKM2rVrG5cvX7YZf62uP/4Wubm5RpUqVYyoqKhCvwMAoDBmpAAAf2nTpk1ydHTU888/b9M+duxYGYahzZs327QbhqERI0Zo7ty5+s9//qOoqChr37Zt23ThwgUNGDBAP//8s3VxdHRU69attXPnzkLHf+6552zW27dvr++///4v616yZInmz5+vWrVqae3atRo3bpwaNmyoTp066aeffjLzE9gICQlRWFiYdb1169aSpI4dO6pGjRqF2q9X69///nfrn318fFS/fn15eHioX79+1vb69evLx8fHZntXV1frM075+fn65Zdf5Onpqfr16+vrr78udJyoqCi5ubnZtPXr108VKlSwmZX69NNP9fPPP+tvf/tb0X4EALjLEaQAAH/p9OnTqlatmipWrGjTfu0tfqdPn7ZpX7ZsmRYsWKC33npLAwYMsOk7ceKEpN9Dh5+fn82ydetWZWRk2IyvUKGC/Pz8bNoqVaqkX3/99S/rdnBwUHR0tA4cOKCff/5Z69evV7du3bRjxw49+eSTRTv56/hjWJIkb29vSVJQUNB12/9c6/XOydvbW9WrVy/0rJa3t7fN9gUFBZozZ47q1q0rV1dXValSRX5+fjp8+LAyMzML1VqrVq1CbT4+PurZs6fN81fLly/XPffco44dO97wvAEA/8XrzwEAJa5t27ZKTEzU/Pnz1a9fP/n6+lr7rj0v9P777yswMLDQtk5Otv80OTo6lkhNlStX1qOPPqpHH31UHTp0UHx8vE6fPq3g4ODrvmhC0g1faHGjmm7UbvzphRy3sv306dP1yiuv6Omnn9a0adPk6+srBwcHjR49utCzWJIKzUZdM2jQIK1evVp79uxRaGioPv74Yw0fPpw3+gFAERGkAAB/KTg4WNu3b9fFixdtZqW+/fZba/8f1alTRzNnzlSHDh3UtWtXxcXFWbe79957JUn+/v7q3LlzGZ2BrZYtWyo+Pl6pqakKDg5WpUqVJEkXLlyQj4+PddyfZ9rKg48++kgPP/yw3n33XZv2CxcuqEqVKkXeT9euXeXn56fly5erdevWunz5sgYOHFjS5QLAHYv/7QQA+Evdu3dXfn5+oW9DzZkzRxaLRd26dSu0TZMmTbRp0yYdO3ZMPXv21G+//SZJCg8Pl5eXl6ZPn668vLxC2507d65Eak5LS9PRo0cLtefm5iouLk4ODg6qU6eOpP+Gu88++8w67tKlS1q6dGmJ1FKSHB0dC81wrV692vQzX05OThowYIA+/PBDxcbGKjQ0VE2aNCnJUgHgjsaMFADgL/Xs2VMPP/yw/vnPf+qHH35Q06ZNtXXrVq1fv16jR4+2BpE/a9OmjdavX6/u3bvr8ccf17p16+Tl5aW3335bAwcOVPPmzfXkk0/Kz89PKSkp2rhxo9q2bVsiH/P98ccf1apVK3Xs2FGdOnVSYGCgMjIytHLlSh06dEijR4+2zuB06dJFNWrU0NChQzV+/Hg5Ojrqvffes9ZVnvTo0UNTp07VkCFD9MADDygpKUnLly9X7dq1Te9r0KBBmjdvnnbu3KnXXnutFKoFgDsXQQoA8JccHBz08ccfa+LEifrggw+0ZMkS1axZU6+//rrGjh170207duyoDz/8UH379tXAgQO1YsUKPfXUU6pWrZpmzJih119/XTk5ObrnnnvUvn17DRkypERqrl+/vt58801t2rRJCxcuVHp6uipUqKDGjRvrnXfe0dChQ61jnZ2dtXbtWg0fPlyvvPKKAgMDNXr0aFWqVKnE6ikp/+///T9dunRJK1as0AcffKDmzZtr48aNeumll0zvq0WLFmrUqJGOHTumyMjIUqgWAO5cFuPP9wcAAIC7xn333SdfX1/FxcXZuxQAuK3wjBQAAHep/fv3KzExUYMGDbJ3KQBw22FGCgCAu8w333yjAwcOaNasWfr555/1/fffq0KFCvYuCwBuK8xIAQBwl/noo480ZMgQ5eXlaeXKlYQoACgGZqQAAAAAwCRmpAAAAADAJIIUAAAAAJjEd6QkFRQU6OzZs6pYsaIsFou9ywEAAABgJ4Zh6OLFi6pWrZocHG4870SQknT27FkFBQXZuwwAAAAA5cSZM2dUvXr1G/YTpCRVrFhR0u8/lpeXl52rAQAAAGAvWVlZCgoKsmaEGyFISdbb+by8vAhSAAAAAP7ykR9eNgEAAAAAJhGkAAAAAMAkghQAAAAAmMQzUgAAACj3DMPQ1atXlZ+fb+9ScJtzdHSUk5PTLX/2iCAFAACAci03N1epqam6fPmyvUvBHcLd3V1Vq1aVi4tLsfdBkAIAAEC5VVBQoOTkZDk6OqpatWpycXG55ZkE3L0Mw1Bubq7OnTun5ORk1a1b96Yf3b0ZghQAAADKrdzcXBUUFCgoKEju7u72Lgd3ADc3Nzk7O+v06dPKzc1VhQoVirUfXjYBAACAcq+4swbA9ZTE9cQVCQAAAAAmEaQAAAAAwCSekQIAAMBtacKapDI9Xkyf0DI9XnlWs2ZNjR49WqNHj7Z3KXbDjBQAAABQCgYPHqzevXuX+XFjY2Pl4+Nz0zEdOnSQxWK54dKhQ4cyqfV2xowUAAAAcJdZs2aNcnNzJUlnzpxRq1attH37djVq1EiSbun7SncLZqQAAACAMtChQwc9//zz+sc//iFfX18FBgZq8uTJNmMsFovefvttdevWTW5ubqpdu7Y++ugja/+uXbtksVh04cIFa1tiYqIsFot++OEH7dq1S0OGDFFmZqZ1dunPx5BkPX5gYKD8/PwkSZUrV7a27dy5U40aNZKrq6tq1qypWbNm3fTc/v3vf8vHx0dxcXGSpG+++UbdunWTp6enAgICNHDgQP38889F/i0Mw9DkyZNVo0YNubq6qlq1anr++eeL+EuXDYIUAAAAUEaWLl0qDw8Pffnll5o5c6amTp2qbdu22Yx55ZVX1LdvXx06dEiRkZF68skndezYsSLt/4EHHtCbb74pLy8vpaamKjU1VePGjTNV44EDB9SvXz89+eSTSkpK0uTJk/XKK68oNjb2uuNnzpypl156SVu3blWnTp104cIFdezYUffdd5/279+vLVu2KD09Xf369Svyb/F///d/mjNnjhYvXqwTJ05o3bp1Cg0tX8+ocWsfAAAAUEaaNGmiSZMmSZLq1q2r+fPnKy4uTo888oh1zBNPPKG///3vkqRp06Zp27Zteuutt7Rw4cK/3L+Li4u8vb1lsVgUGBhYrBpnz56tTp066ZVXXpEk1atXT0ePHtXrr7+uwYMH24x98cUX9f777ys+Pt56W+D8+fN13333afr06dZx7733noKCgvTdd9+pXr16f/lbpKSkKDAwUJ07d5azs7Nq1KihVq1aFet8SgszUgAAAEAZadKkic161apVlZGRYdMWFhZWaL2oM1Il4dixY2rbtq1NW9u2bXXixAnl5+db22bNmqV33nlHX3zxhTVESdKhQ4e0c+dOeXp6WpcGDRpIkk6dOmUdd7Pf4oknntBvv/2m2rVr65lnntHatWt19erVEj/XW0GQAgAAAMqIs7OzzbrFYlFBQUGRt3dw+P0/3w3DsLbl5eWVTHEmtW/fXvn5+frwww9t2rOzs9WzZ08lJibaLCdOnNCDDz5oHXez3yIoKEjHjx/XwoUL5ebmpuHDh+vBBx+027leD0EKAAAAKEf27t1baL1hw4aSZH0xRGpqqrU/MTHRZryLi4vNzJFZDRs21O7du23adu/erXr16snR0dHa1qpVK23evFnTp0/XG2+8YW1v3ry5jhw5opo1a6pOnTo2i4eHR5HrcHNzU8+ePTVv3jzt2rVLCQkJSkoq22+H3QzPSAEAcM0no+xdQfnVc669KwDuGqtXr1bLli3Vrl07LV++XF999ZXeffddSVKdOnUUFBSkyZMn61//+pe+++67Qm/Uq1mzprKzsxUXF6emTZvK3d1d7u7uRT7+2LFjdf/992vatGnq37+/EhISNH/+/Os+o/XAAw9o06ZN6tatm5ycnDR69GhFR0frnXfe0YABA6xv5Tt58qRWrVqlf//73zZh7EZiY2OVn5+v1q1by93dXf/5z3/k5uam4ODgIp9HaSNIAQAA4LYU06d8vcWtpEyZMkWrVq3S8OHDVbVqVa1cuVIhISGSfr8dbuXKlRo2bJiaNGmi+++/X6+++qqeeOIJ6/YPPPCAnnvuOfXv31+//PKLJk2adN1XoN9I8+bN9eGHH2rixImaNm2aqlatqqlTpxZ60cQ17dq108aNG9W9e3c5Ojpq5MiR2r17t1588UV16dJFOTk5Cg4OVteuXa23Jv4VHx8fzZgxQ2PGjFF+fr5CQ0P1ySefqHLlykU+j9JmMf54g+VdKisrS97e3srMzJSXl5e9ywEA2AszUjfGjBTs5MqVK0pOTlatWrVUoUIFe5dT6iwWi9auXavevXvbu5Q72s2uq6JmA56RAgAAAACTCFIAAAAAYBLPSAEAAADlBE/d3D6YkQIAAAAAkwhSAAAAAGASQQoAAAAATCJIAQAAAIBJBCkAAAAAMIkgBQAAAAAm8fpzAAAA3J4+GVW2x+s5t2yPdweZPHmy1q1bp8TERHuXUmKYkQIAAABKyZkzZ/T000+rWrVqcnFxUXBwsEaNGqVffvnF1H5++OEHWSyWUgsiFotF69atu2F/bGysLBbLTZcffvihVGorrwhSAAAAQCn4/vvv1bJlS504cUIrV67UyZMntWjRIsXFxSksLEznz5+3d4lF1r9/f6WmplqXsLAwPfPMMzZtQUFB9i6zTBGkAAAAgFIQHR0tFxcXbd26VQ899JBq1Kihbt26afv27frpp5/0z3/+0zr2ejNCPj4+io2NlSTVqlVLknTffffJYrGoQ4cOkqTBgwerd+/emjJlivz8/OTl5aXnnntOubm51v3UrFlTb775ps2+mzVrpsmTJ1v7Jemxxx6TxWKxrv+Rm5ubAgMDrYuLi4vc3d2t67m5uerTp488PT3l5eWlfv36KT09/Ya/zalTp1S7dm2NGDFChmEoJydH48aN0z333CMPDw+1bt1au3btso6PjY2Vj4+PPv30UzVs2FCenp7q2rWrUlNTrWN27dqlVq1aycPDQz4+Pmrbtq1Onz59wxpuFUEKAAAAKGHnz5/Xp59+quHDh8vNzc2mLzAwUJGRkfrggw9kGEaR9vfVV19JkrZv367U1FStWbPG2hcXF6djx45p165dWrlypdasWaMpU6YUudZ9+/ZJkpYsWaLU1FTrelEVFBSoV69eOn/+vOLj47Vt2zZ9//336t+//3XHHz58WO3atdNTTz2l+fPny2KxaMSIEUpISNCqVat0+PBhPfHEE+ratatOnDhh3e7y5ct644039P777+uzzz5TSkqKxo0bJ0m6evWqevfurYceekiHDx9WQkKCnn32WVksFlPnYgYvmwAAAABK2IkTJ2QYhho2bHjd/oYNG+rXX3/VuXPn5O/v/5f78/PzkyRVrlxZgYGBNn0uLi5677335O7urkaNGmnq1KkaP368pk2bJgeHv543ubZvHx+fQvsuiri4OCUlJSk5Odl6e9+yZcvUqFEj7du3T/fff7917J49e9SjRw/985//1NixYyVJKSkpWrJkiVJSUlStWjVJ0rhx47RlyxYtWbJE06dPlyTl5eVp0aJFuvfeeyVJI0aM0NSpUyVJWVlZyszMVI8ePaz9N/rtSwozUgAAAEApKeqM061o2rSp3N3drethYWHKzs7WmTNnSv3YknTs2DEFBQXZPCMVEhIiHx8fHTt2zNqWkpKiRx55RBMnTrSGKElKSkpSfn6+6tWrJ09PT+sSHx+vU6dOWce5u7tbQ5IkVa1aVRkZGZIkX19fDR48WOHh4erZs6fmzp1rc9tfaSBIAQAAACWsTp06slgsNkHij44dO6ZKlSpZZ4MsFkuh0JWXl1citTg4OJTavs3w8/NTq1attHLlSmVlZVnbs7Oz5ejoqAMHDigxMdG6HDt2THPn/veV887Ozjb7+/NvtmTJEiUkJOiBBx7QBx98oHr16mnv3r2ldj4EKQAAAKCEVa5cWY888ogWLlyo3377zaYvLS1Ny5cvV//+/a3P8Pj5+dnMoJw4cUKXL1+2rru4uEiS8vPzCx3r0KFDNsfYu3evPD09rTNEf953VlaWkpOTbfbh7Ox83X0XRcOGDXXmzBmbGbCjR4/qwoULCgkJsba5ublpw4YNqlChgsLDw3Xx4kVJv79AIz8/XxkZGapTp47NYvZWw/vuu08TJkzQnj171LhxY61YsaJY51QUBCkAAACgFMyfP185OTkKDw/XZ599pjNnzmjLli165JFHdM899+hf//qXdWzHjh01f/58HTx4UPv379dzzz1nMwPj7+8vNzc3bdmyRenp6crMzLT25ebmaujQoTp69Kg2bdqkSZMmacSIEdbnozp27Kj3339fn3/+uZKSkhQVFSVHR0ebWmvWrKm4uDilpaXp119/NXWenTt3VmhoqCIjI/X111/rq6++0qBBg/TQQw+pZcuWNmM9PDy0ceNGOTk5qVu3bsrOzla9evUUGRmpQYMGac2aNUpOTtZXX32lmJgYbdy4sUg1JCcna8KECUpISNDp06e1detWnThxolSfk+JlEwAAALg99Zz712PsqG7dutq/f78mTZqkfv366fz58woMDFTv3r01adIk+fr6WsfOmjVLQ4YMUfv27VWtWjXNnTtXBw4csPY7OTlp3rx5mjp1qiZOnKj27dtbXw/eqVMn1a1bVw8++KBycnI0YMAA66vNJWnChAlKTk5Wjx495O3trWnTphWakZo1a5bGjBmjd955R/fcc4+pj+taLBatX79eI0eO1IMPPigHBwd17dpVb7311nXHe3p6avPmzQoPD1dERIQ2bdqkJUuW6NVXX9XYsWP1008/qUqVKmrTpo169OhRpBrc3d317bffaunSpfrll19UtWpVRUdH63/+53+KfB5mWYyyeAKunMvKypK3t7cyMzPl5eVl73IAAPbyySh7V1B+lfP/YMWd68qVK0pOTlatWrVUoUIFe5dT7gwePFgXLlwo9A0q3NzNrquiZgNu7QMAAAAAkwhSAAAAAGASz0gBAAAAt6nY2Fh7l3DXYkYKAAAAAEwiSAEAAKDc4/1oKEklcT0RpAAAAFBuXfuW0h8/TgvcqmvX0x+/1WUWz0gBAACg3HJ0dJSPj48yMjIk/f69IIvFYueqcLsyDEOXL19WRkaGfHx8Cn2Y2AyCFAAAAMq1wMBASbKGKeBW+fj4WK+r4iJIAQAAoFyzWCyqWrWq/P39lZeXZ+9ycJtzdna+pZmoawhSKJcmrEm6aX9Mn9AyqgQAAJQXjo6OJfIfwEBJ4GUTAAAAAGASQQoAAAAATCJIAQAAAIBJBCkAAAAAMIkgBQAAAAAmEaQAAAAAwCSCFAAAAACYRJACAAAAAJPsGqQmT54si8ViszRo0MDaf+XKFUVHR6ty5cry9PRU3759lZ6ebrOPlJQURUREyN3dXf7+/ho/fryuXr1a1qcCAAAA4C7iZO8CGjVqpO3bt1vXnZz+W9ILL7ygjRs3avXq1fL29taIESPUp08f7d69W5KUn5+viIgIBQYGas+ePUpNTdWgQYPk7Oys6dOnl/m5AAAAALg72D1IOTk5KTAwsFB7Zmam3n33Xa1YsUIdO3aUJC1ZskQNGzbU3r171aZNG23dulVHjx7V9u3bFRAQoGbNmmnatGl68cUXNXnyZLm4uFz3mDk5OcrJybGuZ2Vllc7JAQAAALgj2f0ZqRMnTqhatWqqXbu2IiMjlZKSIkk6cOCA8vLy1LlzZ+vYBg0aqEaNGkpISJAkJSQkKDQ0VAEBAdYx4eHhysrK0pEjR254zJiYGHl7e1uXoKCgUjo7AAAAAHciuwap1q1bKzY2Vlu2bNHbb7+t5ORktW/fXhcvXlRaWppcXFzk4+Njs01AQIDS0tIkSWlpaTYh6lr/tb4bmTBhgjIzM63LmTNnSvbEAAAAANzR7HprX7du3ax/btKkiVq3bq3g4GB9+OGHcnNzK7Xjurq6ytXVtdT2DwAAAODOZvdb+/7Ix8dH9erV08mTJxUYGKjc3FxduHDBZkx6err1marAwMBCb/G7tn69564AAAAAoCSUqyCVnZ2tU6dOqWrVqmrRooWcnZ0VFxdn7T9+/LhSUlIUFhYmSQoLC1NSUpIyMjKsY7Zt2yYvLy+FhISUef0AAAAA7g52vbVv3Lhx6tmzp4KDg3X27FlNmjRJjo6OGjBggLy9vTV06FCNGTNGvr6+8vLy0siRIxUWFqY2bdpIkrp06aKQkBANHDhQM2fOVFpaml5++WVFR0dz6x4AAACAUmPXIPXjjz9qwIAB+uWXX+Tn56d27dpp79698vPzkyTNmTNHDg4O6tu3r3JychQeHq6FCxdat3d0dNSGDRs0bNgwhYWFycPDQ1FRUZo6daq9TgkAAADAXcBiGIZh7yLsLSsrS97e3srMzJSXl5e9y4GkCWuSbtof0ye0jCoBcFf5ZJS9Kyi/es61dwUAUCaKmg3K1TNSAAAAAHA7IEgBAAAAgEkEKQAAAAAwiSAFAAAAACYRpAAAAADAJIIUAAAAAJhEkAIAAAAAkwhSAAAAAGASQQoAAAAATCJIAQAAAIBJBCkAAAAAMIkgBQAAAAAmEaQAAAAAwCSCFAAAAACYRJACAAAAAJMIUgAAAABgEkEKAAAAAEwiSAEAAACASQQpAAAAADCJIAUAAAAAJhGkAAAAAMAkghQAAAAAmESQAgAAAACTCFIAAAAAYBJBCgAAAABMIkgBAAAAgEkEKQAAAAAwiSAFAAAAACYRpAAAAADAJIIUAAAAAJhEkAIAAAAAkwhSAAAAAGASQQoAAAAATCJIAQAAAIBJBCkAAAAAMIkgBQAAAAAmEaQAAAAAwCQnexcAAABuA5+MsncF5VPPufauAICdMCMFAAAAACYRpAAAAADAJIIUAAAAAJhEkAIAAAAAkwhSAAAAAGASQQoAAAAATCJIAQAAAIBJBCkAAAAAMIkgBQAAAAAmEaQAAAAAwCSCFAAAAACYRJACAAAAAJMIUgAAAABgEkEKAAAAAEwiSAEAAACASQQpAAAAADCJIAUAAAAAJhGkAAAAAMAkghQAAAAAmESQAgAAAACTCFIAAAAAYBJBCgAAAABMIkgBAAAAgEkEKQAAAAAwiSAFAAAAACYRpAAAAADAJIIUAAAAAJhEkAIAAAAAkwhSAAAAAGBSuQlSM2bMkMVi0ejRo61tV65cUXR0tCpXrixPT0/17dtX6enpNtulpKQoIiJC7u7u8vf31/jx43X16tUyrh4AAADA3aRcBKl9+/Zp8eLFatKkiU37Cy+8oE8++USrV69WfHy8zp49qz59+lj78/PzFRERodzcXO3Zs0dLly5VbGysJk6cWNanAAAAAOAuYvcglZ2drcjISL3zzjuqVKmStT0zM1PvvvuuZs+erY4dO6pFixZasmSJ9uzZo71790qStm7dqqNHj+o///mPmjVrpm7dumnatGlasGCBcnNz7XVKAAAAAO5wdg9S0dHRioiIUOfOnW3aDxw4oLy8PJv2Bg0aqEaNGkpISJAkJSQkKDQ0VAEBAdYx4eHhysrK0pEjR254zJycHGVlZdksAAAAAFBUTvY8+KpVq/T1119r3759hfrS0tLk4uIiHx8fm/aAgAClpaVZx/wxRF3rv9Z3IzExMZoyZcotVg8AAADgbmW3GakzZ85o1KhRWr58uSpUqFCmx54wYYIyMzOty5kzZ8r0+AAAAABub3YLUgcOHFBGRoaaN28uJycnOTk5KT4+XvPmzZOTk5MCAgKUm5urCxcu2GyXnp6uwMBASVJgYGCht/hdW7825npcXV3l5eVlswAAAABAUdktSHXq1ElJSUlKTEy0Li1btlRkZKT1z87OzoqLi7Nuc/z4caWkpCgsLEySFBYWpqSkJGVkZFjHbNu2TV5eXgoJCSnzcwIAAABwd7DbM1IVK1ZU48aNbdo8PDxUuXJla/vQoUM1ZswY+fr6ysvLSyNHjlRYWJjatGkjSerSpYtCQkI0cOBAzZw5U2lpaXr55ZcVHR0tV1fXMj8nAAAAAHcHu75s4q/MmTNHDg4O6tu3r3JychQeHq6FCxda+x0dHbVhwwYNGzZMYWFh8vDwUFRUlKZOnWrHqgEAAADc6cpVkNq1a5fNeoUKFbRgwQItWLDghtsEBwdr06ZNpVwZAAAAAPxXuQpSQFFNWJN00/6YPqFlVAkAAADuRnb/IC8AAAAA3G4IUgAAAABgEkEKAAAAAEwiSAEAAACASQQpAAAAADCJIAUAAAAAJhGkAAAAAMAkghQAAAAAmESQAgAAAACTCFIAAAAAYBJBCgAAAABMIkgBAAAAgEkEKQAAAAAwiSAFAAAAACYRpAAAAADAJIIUAAAAAJhEkAIAAAAAkwhSAAAAAGASQQoAAAAATCJIAQAAAIBJBCkAAAAAMIkgBQAAAAAmEaQAAAAAwCSCFAAAAACYRJACAAAAAJMIUgAAAABgEkEKAAAAAEwiSAEAAACASQQpAAAAADCJIAUAAAAAJhGkAAAAAMAkghQAAAAAmESQAgAAAACTCFIAAAAAYBJBCgAAAABMIkgBAAAAgEkEKQAAAAAwiSAFAAAAACYRpAAAAADAJIIUAAAAAJhEkAIAAAAAkwhSAAAAAGASQQoAAAAATCJIAQAAAIBJxQpS33//fUnXAQAAAAC3jWIFqTp16ujhhx/Wf/7zH125cqWkawIAAACAcq1YQerrr79WkyZNNGbMGAUGBup//ud/9NVXX5V0bQAAAABQLhUrSDVr1kxz587V2bNn9d577yk1NVXt2rVT48aNNXv2bJ07d66k6wQAAACAcuOWXjbh5OSkPn36aPXq1Xrttdd08uRJjRs3TkFBQRo0aJBSU1NLqk4AAAAAKDduKUjt379fw4cPV9WqVTV79myNGzdOp06d0rZt23T27Fn16tWrpOoEAAAAgHLDqTgbzZ49W0uWLNHx48fVvXt3LVu2TN27d5eDw++5rFatWoqNjVXNmjVLslYAAAAAKBeKFaTefvttPf300xo8eLCqVq163TH+/v569913b6k4AAAAACiPihWkTpw48ZdjXFxcFBUVVZzdAwAAAEC5VqxnpJYsWaLVq1cXal+9erWWLl16y0UBAAAAQHlWrCAVExOjKlWqFGr39/fX9OnTb7koAAAAACjPinVrX0pKimrVqlWoPTg4WCkpKbdcFACglH0yyt4VAABwWyvWjJS/v78OHz5cqP3QoUOqXLnyLRcFAAAAAOVZsYLUgAED9Pzzz2vnzp3Kz89Xfn6+duzYoVGjRunJJ58s6RoBAAAAoFwp1q1906ZN0w8//KBOnTrJyen3XRQUFGjQoEE8IwUAAADgjlesIOXi4qIPPvhA06ZN06FDh+Tm5qbQ0FAFBweXdH24Q01Yk2TvEgAAAIBiK1aQuqZevXqqV69eSdUCAAAAALeFYgWp/Px8xcbGKi4uThkZGSooKLDp37FjR4kUBwAAAADlUbGC1KhRoxQbG6uIiAg1btxYFoulpOsCAAAAgHKrWEFq1apV+vDDD9W9e/eSrgcAAAAAyr1ivf7cxcVFderUKelaAAAAAOC2UKwgNXbsWM2dO1eGYZR0PQAAAABQ7hXr1r4vvvhCO3fu1ObNm9WoUSM5Ozvb9K9Zs6ZEigMAAACA8qhYM1I+Pj567LHH9NBDD6lKlSry9va2WYrq7bffVpMmTeTl5SUvLy+FhYVp8+bN1v4rV64oOjpalStXlqenp/r27av09HSbfaSkpCgiIkLu7u7y9/fX+PHjdfXq1eKcFgAAAAAUSbFmpJYsWVIiB69evbpmzJihunXryjAMLV26VL169dLBgwfVqFEjvfDCC9q4caNWr14tb29vjRgxQn369NHu3bsl/f4a9oiICAUGBmrPnj1KTU3VoEGD5OzsrOnTp5dIjQAAAADwZxajmA86Xb16Vbt27dKpU6f01FNPqWLFijp79qy8vLzk6elZ7IJ8fX31+uuv6/HHH5efn59WrFihxx9/XJL07bffqmHDhkpISFCbNm20efNm9ejRQ2fPnlVAQIAkadGiRXrxxRd17tw5ubi4FOmYWVlZ8vb2VmZmpry8vIpdO4puwpqkUt1/TJ/QUt0/cNv7ZJS9KwDuDD3n2rsCACWsqNmgWLf2nT59WqGhoerVq5eio6N17tw5SdJrr72mcePGFavg/Px8rVq1SpcuXVJYWJgOHDigvLw8de7c2TqmQYMGqlGjhhISEiRJCQkJCg0NtYYoSQoPD1dWVpaOHDlyw2Pl5OQoKyvLZgEAAACAoipWkBo1apRatmypX3/9VW5ubtb2xx57THFxcab2lZSUJE9PT7m6uuq5557T2rVrFRISorS0NLm4uMjHx8dmfEBAgNLS0iRJaWlpNiHqWv+1vhuJiYmxeaYrKCjIVM0AAAAA7m7Fekbq888/1549ewrdOlezZk399NNPpvZVv359JSYmKjMzUx999JGioqIUHx9fnLKKbMKECRozZox1PSsrizAFAAAAoMiKFaQKCgqUn59fqP3HH39UxYoVTe3rjx/3bdGihfbt26e5c+eqf//+ys3N1YULF2xmpdLT0xUYGChJCgwM1FdffWWzv2tv9bs25npcXV3l6upqqk4AAAAAuKZYt/Z16dJFb775pnXdYrEoOztbkyZNUvfu3W+poIKCAuXk5KhFixZydna2uVXw+PHjSklJUVhYmCQpLCxMSUlJysjIsI7Ztm2bvLy8FBISckt1AAAAAMCNFGtGatasWQoPD1dISIiuXLmip556SidOnFCVKlW0cuXKIu9nwoQJ6tatm2rUqKGLFy9qxYoV2rVrlz799FN5e3tr6NChGjNmjHx9feXl5aWRI0cqLCxMbdq0kfR7oAsJCdHAgQM1c+ZMpaWl6eWXX1Z0dDQzTgAAAABKTbGCVPXq1XXo0CGtWrVKhw8fVnZ2toYOHarIyEibl0/8lYyMDA0aNEipqany9vZWkyZN9Omnn+qRRx6RJM2ZM0cODg7q27evcnJyFB4eroULF1q3d3R01IYNGzRs2DCFhYXJw8NDUVFRmjp1anFOCwAAAACKpNjfkbqT8B2pssd3pAA74ztSQMngO1LAHaeo2aBYM1LLli27af+gQYOKs1sAAAAAuC0UK0iNGmX7fzLz8vJ0+fJlubi4yN3dnSAFAAAA4I5WrLf2/frrrzZLdna2jh8/rnbt2pl62QQAAAAA3I6KFaSup27dupoxY0ah2SoAAAAAuNOUWJCSJCcnJ509e7YkdwkAAAAA5U6xnpH6+OOPbdYNw1Bqaqrmz5+vtm3blkhhAAAAAFBeFStI9e7d22bdYrHIz89PHTt21KxZs0qiLgAAAAAot4oVpAoKCkq6DgAAAAC4bZToM1IAAAAAcDco1ozUmDFjijx29uzZxTkEAAAAAJRbxQpSBw8e1MGDB5WXl6f69etLkr777js5OjqqefPm1nEWi6VkqgQAAACAcqRYQapnz56qWLGili5dqkqVKkn6/SO9Q4YMUfv27TV27NgSLRIAAAAAypNiPSM1a9YsxcTEWEOUJFWqVEmvvvoqb+0DAAAAcMcrVpDKysrSuXPnCrWfO3dOFy9evOWiAAAAAKA8K1aQeuyxxzRkyBCtWbNGP/74o3788Uf93//9n4YOHao+ffqUdI0AAAAAUK4U6xmpRYsWady4cXrqqaeUl5f3+46cnDR06FC9/vrrJVogAAAAAJQ3xQpS7u7uWrhwoV5//XWdOnVKknTvvffKw8OjRIsDAAAAgPLolj7Im5qaqtTUVNWtW1ceHh4yDKOk6gIAAACAcqtYQeqXX35Rp06dVK9ePXXv3l2pqamSpKFDh/LqcwAAAAB3vGIFqRdeeEHOzs5KSUmRu7u7tb1///7asmVLiRUHAAAAAOVRsZ6R2rp1qz799FNVr17dpr1u3bo6ffp0iRQGAAAAAOVVsWakLl26ZDMTdc358+fl6up6y0UBAAAAQHlWrCDVvn17LVu2zLpusVhUUFCgmTNn6uGHHy6x4gAAAACgPCrWrX0zZ85Up06dtH//fuXm5uof//iHjhw5ovPnz2v37t0lXSMAAAAAlCvFmpFq3LixvvvuO7Vr1069evXSpUuX1KdPHx08eFD33ntvSdcIAAAAAOWK6RmpvLw8de3aVYsWLdI///nP0qgJAAAAAMo10zNSzs7OOnz4cGnUAgAAAAC3hWLd2ve3v/1N7777bknXAgAAAAC3hWK9bOLq1at67733tH37drVo0UIeHh42/bNnzy6R4gAAAACgPDIVpL7//nvVrFlT33zzjZo3by5J+u6772zGWCyWkqsOAAAAAMohU0Gqbt26Sk1N1c6dOyVJ/fv317x58xQQEFAqxQEAAABAeWTqGSnDMGzWN2/erEuXLpVoQQAAAABQ3hXrZRPX/DlYAQAAAMDdwFSQslgshZ6B4pkoAAAAAHcbU89IGYahwYMHy9XVVZJ05coVPffcc4Xe2rdmzZqSqxAAAAAAyhlTQSoqKspm/W9/+1uJFgMAAAAAtwNTQWrJkiWlVQcAAAAA3DZu6WUTAAAAAHA3IkgBAAAAgEkEKQAAAAAwiSAFAAAAACYRpAAAAADAJIIUAAAAAJhEkAIAAAAAkwhSAAAAAGASQQoAAAAATCJIAQAAAIBJBCkAAAAAMIkgBQAAAAAmEaQAAAAAwCSCFAAAAACYRJACAAAAAJMIUgAAAABgEkEKAAAAAEwiSAEAAACASQQpAAAAADCJIAUAAAAAJhGkAAAAAMAkghQAAAAAmESQAgAAAACTCFIAAAAAYBJBCgAAAABMIkgBAAAAgEkEKQAAAAAwiSAFAAAAACYRpAAAAADAJIIUAAAAAJhEkAIAAAAAk+wapGJiYnT//ferYsWK8vf3V+/evXX8+HGbMVeuXFF0dLQqV64sT09P9e3bV+np6TZjUlJSFBERIXd3d/n7+2v8+PG6evVqWZ4KAAAAgLuIXYNUfHy8oqOjtXfvXm3btk15eXnq0qWLLl26ZB3zwgsv6JNPPtHq1asVHx+vs2fPqk+fPtb+/Px8RUREKDc3V3v27NHSpUsVGxuriRMn2uOUAAAAANwFLIZhGPYu4ppz587J399f8fHxevDBB5WZmSk/Pz+tWLFCjz/+uCTp22+/VcOGDZWQkKA2bdpo8+bN6tGjh86ePauAgABJ0qJFi/Tiiy/q3LlzcnFx+cvjZmVlydvbW5mZmfLy8irVc8TvJqxJKtX9x/QJLdX9A7e9T0bZuwLgztBzrr0rAFDCipoNytUzUpmZmZIkX19fSdKBAweUl5enzp07W8c0aNBANWrUUEJCgiQpISFBoaGh1hAlSeHh4crKytKRI0eue5ycnBxlZWXZLAAAAABQVOUmSBUUFGj06NFq27atGjduLElKS0uTi4uLfHx8bMYGBAQoLS3NOuaPIepa/7W+64mJiZG3t7d1CQoKKuGzAQAAAHAnKzdBKjo6Wt98841WrVpV6seaMGGCMjMzrcuZM2dK/ZgAAAAA7hxO9i5AkkaMGKENGzbos88+U/Xq1a3tgYGBys3N1YULF2xmpdLT0xUYGGgd89VXX9ns79pb/a6N+TNXV1e5urqW8FkAAAAAuFvYdUbKMAyNGDFCa9eu1Y4dO1SrVi2b/hYtWsjZ2VlxcXHWtuPHjyslJUVhYWGSpLCwMCUlJSkjI8M6Ztu2bfLy8lJISEjZnAgAAACAu4pdZ6Sio6O1YsUKrV+/XhUrVrQ+0+Tt7S03Nzd5e3tr6NChGjNmjHx9feXl5aWRI0cqLCxMbdq0kSR16dJFISEhGjhwoGbOnKm0tDS9/PLLio6OZtYJAACULt6AeWO80RB3OLsGqbfffluS1KFDB5v2JUuWaPDgwZKkOXPmyMHBQX379lVOTo7Cw8O1cOFC61hHR0dt2LBBw4YNU1hYmDw8PBQVFaWpU6eW1WkAAAAAuMvYNUgV5RNWFSpU0IIFC7RgwYIbjgkODtamTZtKsjQAAAAAuKFy89Y+AAAAALhdEKQAAAAAwKRy8fpzoKRNWJP0l2Ni+oSWQSUAAAC4EzEjBQAAAAAmEaQAAAAAwCSCFAAAAACYRJACAAAAAJMIUgAAAABgEkEKAAAAAEwiSAEAAACASQQpAAAAADCJIAUAAAAAJhGkAAAAAMAkghQAAAAAmESQAgAAAACTCFIAAAAAYBJBCgAAAABMIkgBAAAAgEkEKQAAAAAwiSAFAAAAACYRpAAAAADAJIIUAAAAAJhEkAIAAAAAkwhSAAAAAGASQQoAAAAATCJIAQAAAIBJBCkAAAAAMIkgBQAAAAAmEaQAAAAAwCSCFAAAAACYRJACAAAAAJMIUgAAAABgEkEKAAAAAEwiSAEAAACASQQpAAAAADCJIAUAAAAAJhGkAAAAAMAkghQAAAAAmESQAgAAAACTCFIAAAAAYBJBCgAAAABMIkgBAAAAgEkEKQAAAAAwiSAFAAAAACYRpAAAAADAJIIUAAAAAJhEkAIAAAAAkwhSAAAAAGASQQoAAAAATCJIAQAAAIBJBCkAAAAAMIkgBQAAAAAmEaQAAAAAwCSCFAAAAACYRJACAAAAAJMIUgAAAABgkpO9CwDsZcKapJv2x/QJLaNKAAAAcLthRgoAAAAATCJIAQAAAIBJBCkAAAAAMIkgBQAAAAAmEaQAAAAAwCSCFAAAAACYRJACAAAAAJMIUgAAAABgEkEKAAAAAEwiSAEAAACASQQpAAAAADDJrkHqs88+U8+ePVWtWjVZLBatW7fOpt8wDE2cOFFVq1aVm5ubOnfurBMnTtiMOX/+vCIjI+Xl5SUfHx8NHTpU2dnZZXgWAAAAAO42dg1Sly5dUtOmTbVgwYLr9s+cOVPz5s3TokWL9OWXX8rDw0Ph4eG6cuWKdUxkZKSOHDmibdu2acOGDfrss8/07LPPltUpAAAAALgLOdnz4N26dVO3bt2u22cYht588029/PLL6tWrlyRp2bJlCggI0Lp16/Tkk0/q2LFj2rJli/bt26eWLVtKkt566y11795db7zxhqpVq1Zm5wIAAADg7lFun5FKTk5WWlqaOnfubG3z9vZW69atlZCQIElKSEiQj4+PNURJUufOneXg4KAvv/zyhvvOyclRVlaWzQIAAAAARVVug1RaWpokKSAgwKY9ICDA2peWliZ/f3+bficnJ/n6+lrHXE9MTIy8vb2tS1BQUAlXDwAAAOBOVm6DVGmaMGGCMjMzrcuZM2fsXRIAAACA20i5DVKBgYGSpPT0dJv29PR0a19gYKAyMjJs+q9evarz589bx1yPq6urvLy8bBYAAAAAKKpyG6Rq1aqlwMBAxcXFWduysrL05ZdfKiwsTJIUFhamCxcu6MCBA9YxO3bsUEFBgVq3bl3mNQMAAAC4O9j1rX3Z2dk6efKkdT05OVmJiYny9fVVjRo1NHr0aL366quqW7euatWqpVdeeUXVqlVT7969JUkNGzZU165d9cwzz2jRokXKy8vTiBEj9OSTT/LGPgAAAAClxq5Bav/+/Xr44Yet62PGjJEkRUVFKTY2Vv/4xz906dIlPfvss7pw4YLatWunLVu2qEKFCtZtli9frhEjRqhTp05ycHBQ3759NW/evDI/FwAAAAB3D4thGIa9i7C3rKwseXt7KzMzk+elysiENUn2LuEvxfQJtXcJQOn5ZJS9KwBwp+s5194VAMVS1GxQbp+RAgAAAIDyiiAFAAAAACYRpAAAAADAJIIUAAAAAJhEkAIAAAAAkwhSAAAAAGASQQoAAAAATCJIAQAAAIBJBCkAAAAAMIkgBQAAAAAmEaQAAAAAwCSCFAAAAACYRJACAAAAAJOc7F0AUF5NWJN00/6YPqFlVAkAAADKG2akAAAAAMAkZqQAAABQ8j4ZZe8Kyqeec+1dAUoIM1IAAAAAYBJBCgAAAABM4tY+AHcubisBAAClhBkpAAAAADCJIAUAAAAAJhGkAAAAAMAkghQAAAAAmESQAgAAAACTCFIAAAAAYBJBCgAAAABMIkgBAAAAgEkEKQAAAAAwiSAFAAAAACYRpAAAAADAJIIUAAAAAJhEkAIAAAAAkwhSAAAAAGASQQoAAAAATCJIAQAAAIBJTvYuAHemCWuS7F0CAAAAUGqYkQIAAAAAkwhSAAAAAGASQQoAAAAATCJIAQAAAIBJBCkAAAAAMIkgBQAAAAAmEaQAAAAAwCSCFAAAAACYRJACAAAAAJMIUgAAAABgEkEKAAAAAEwiSAEAAACASQQpAAAAADCJIAUAAAAAJhGkAAAAAMAkghQAAAAAmESQAgAAAACTCFIAAAAAYJKTvQsAblcT1iTdtD+mT2gZVQIAAICyxowUAAAAAJhEkAIAAAAAkwhSAAAAAGASz0gBAAAAZeWTUfauoHzqOdfeFZjGjBQAAAAAmESQAgAAAACTCFIAAAAAYBJBCgAAAABMIkgBAAAAgEkEKQAAAAAwiSAFAAAAACYRpAAAAADAJD7IC5SSCWuSbtof0ye0jCoBAABASbtjgtSCBQv0+uuvKy0tTU2bNtVbb72lVq1a2bssoGzwlXQAAIAydUcEqQ8++EBjxozRokWL1Lp1a7355psKDw/X8ePH5e/vb+/ygOtixgoAAOD2dUc8IzV79mw988wzGjJkiEJCQrRo0SK5u7vrvffes3dpAAAAAO5At/2MVG5urg4cOKAJEyZY2xwcHNS5c2clJCRcd5ucnBzl5ORY1zMzMyVJWVlZpVtsUW3+h70ruGWPnP7V3iXc9rI+qGTvEgAAAMpGefnvcP03ExiGcdNxt32Q+vnnn5Wfn6+AgACb9oCAAH377bfX3SYmJkZTpkwp1B4UFFQqNQIAAAC4mcX2LqCQixcvytvb+4b9t32QKo4JEyZozJgx1vWCggKdP39elStXlsVisWNlt5esrCwFBQXpzJkz8vLysnc5KOe4XmAG1wvM4pqBGVwvuBnDMHTx4kVVq1btpuNu+yBVpUoVOTo6Kj093aY9PT1dgYGB193G1dVVrq6uNm0+Pj6lVeIdz8vLi7+EUGRcLzCD6wVmcc3ADK4X3MjNZqKuue1fNuHi4qIWLVooLi7O2lZQUKC4uDiFhYXZsTIAAAAAd6rbfkZKksaMGaOoqCi1bNlSrVq10ptvvqlLly5pyJAh9i4NAAAAwB3ojghS/fv317lz5zRx4kSlpaWpWbNm2rJlS6EXUKBkubq6atKkSYVukwSuh+sFZnC9wCyuGZjB9YKSYDH+6r1+AAAAAAAbt/0zUgAAAABQ1ghSAAAAAGASQQoAAAAATCJIAQAAAIBJBCkU8tlnn6lnz56qVq2aLBaL1q1bZ9NvGIYmTpyoqlWrys3NTZ07d9aJEydsxpw/f16RkZHy8vKSj4+Phg4dquzs7DI8C5SFmJgY3X///apYsaL8/f3Vu3dvHT9+3GbMlStXFB0drcqVK8vT01N9+/Yt9AHtlJQURUREyN3dXf7+/ho/fryuXr1alqeCMvD222+rSZMm1g9ghoWFafPmzdZ+rhXczIwZM2SxWDR69GhrG9cM/mjy5MmyWCw2S4MGDaz9XC8oaQQpFHLp0iU1bdpUCxYsuG7/zJkzNW/ePC1atEhffvmlPDw8FB4eritXrljHREZG6siRI9q2bZs2bNigzz77TM8++2xZnQLKSHx8vKKjo7V3715t27ZNeXl56tKliy5dumQd88ILL+iTTz7R6tWrFR8fr7Nnz6pPnz7W/vz8fEVERCg3N1d79uzR0qVLFRsbq4kTJ9rjlFCKqlevrhkzZujAgQPav3+/OnbsqF69eunIkSOSuFZwY/v27dPixYvVpEkTm3auGfxZo0aNlJqaal2++OILax/XC0qcAdyEJGPt2rXW9YKCAiMwMNB4/fXXrW0XLlwwXF1djZUrVxqGYRhHjx41JBn79u2zjtm8ebNhsViMn376qcxqR9nLyMgwJBnx8fGGYfx+bTg7OxurV6+2jjl27JghyUhISDAMwzA2bdpkODg4GGlpadYxb7/9tuHl5WXk5OSU7QmgzFWqVMn497//zbWCG7p48aJRt25dY9u2bcZDDz1kjBo1yjAM/n5BYZMmTTKaNm163T6uF5QGZqRgSnJystLS0tS5c2drm7e3t1q3bq2EhARJUkJCgnx8fNSyZUvrmM6dO8vBwUFffvllmdeMspOZmSlJ8vX1lSQdOHBAeXl5NtdLgwYNVKNGDZvrJTQ01OYD2uHh4crKyrLOVODOk5+fr1WrVunSpUsKCwvjWsENRUdHKyIiwubakPj7Bdd34sQJVatWTbVr11ZkZKRSUlIkcb2gdDjZuwDcXtLS0iTJ5i+Za+vX+tLS0uTv72/T7+TkJF9fX+sY3HkKCgo0evRotW3bVo0bN5b0+7Xg4uIiHx8fm7F/vl6udz1d68OdJSkpSWFhYbpy5Yo8PT21du1ahYSEKDExkWsFhaxatUpff/219u3bV6iPv1/wZ61bt1ZsbKzq16+v1NRUTZkyRe3bt9c333zD9YJSQZACUCKio6P1zTff2NyPDvxZ/fr1lZiYqMzMTH300UeKiopSfHy8vctCOXTmzBmNGjVK27ZtU4UKFexdDm4D3bp1s/65SZMmat26tYKDg/Xhhx/Kzc3NjpXhTsWtfTAlMDBQkgq95SY9Pd3aFxgYqIyMDJv+q1ev6vz589YxuLOMGDFCGzZs0M6dO1W9enVre2BgoHJzc3XhwgWb8X++Xq53PV3rw53FxcVFderUUYsWLRQTE6OmTZtq7ty5XCso5MCBA8rIyFDz5s3l5OQkJycnxcfHa968eXJyclJAQADXDG7Kx8dH9erV08mTJ/k7BqWCIAVTatWqpcDAQMXFxVnbsrKy9OWXXyosLEySFBYWpgsXLujAgQPWMTt27FBBQYFat25d5jWj9BiGoREjRmjt2rXasWOHatWqZdPfokULOTs721wvx48fV0pKis31kpSUZBO+t23bJi8vL4WEhJTNicBuCgoKlJOTw7WCQjp16qSkpCQlJiZal5YtWyoyMtL6Z64Z3Ex2drZOnTqlqlWr8ncMSoe933aB8ufixYvGwYMHjYMHDxqSjNmzZxsHDx40Tp8+bRiGYcyYMcPw8fEx1q9fbxw+fNjo1auXUatWLeO3336z7qNr167GfffdZ3z55ZfGF198YdStW9cYMGCAvU4JpWTYsGGGt7e3sWvXLiM1NdW6XL582TrmueeeM2rUqGHs2LHD2L9/vxEWFmaEhYVZ+69evWo0btzY6NKli5GYmGhs2bLF8PPzMyZMmGCPU0Ipeumll4z4+HgjOTnZOHz4sPHSSy8ZFovF2Lp1q2EYXCv4a398a59hcM3A1tixY41du3YZycnJxu7du43OnTsbVapUMTIyMgzD4HpBySNIoZCdO3cakgotUVFRhmH8/gr0V155xQgICDBcXV2NTp06GcePH7fZxy+//GIMGDDA8PT0NLy8vIwhQ4YYFy9etMPZoDRd7zqRZCxZssQ65rfffjOGDx9uVKpUyXB3dzcee+wxIzU11WY/P/zwg9GtWzfDzc3NqFKlijF27FgjLy+vjM8Gpe3pp582goODDRcXF8PPz8/o1KmTNUQZBtcK/tqfgxTXDP6of//+RtWqVQ0XFxfjnnvuMfr372+cPHnS2s/1gpJmMQzDsM9cGAAAAADcnnhGCgAAAABMIkgBAAAAgEkEKQAAAAAwiSAFAAAAACYRpAAAAADAJIIUAAAAAJhEkAIAAAAAkwhSAAAAAGASQQoAUK798MMPslgsSkxMtHcpAABYEaQAAKXOYrHcdJk8ebK9S7yukydPasiQIapevbpcXV1Vq1YtDRgwQPv37y/TOgiTAFD+ONm7AADAnS81NdX65w8++EATJ07U8ePHrW2enp72KOum9u/fr06dOqlx48ZavHixGjRooIsXL2r9+vUaO3as4uPj7V0iAMCOmJECAJS6wMBA6+Lt7S2LxWJd9/f31+zZs62zPs2aNdOWLVtuuK/8/Hw9/fTTatCggVJSUiRJ69evV/PmzVWhQgXVrl1bU6ZM0dWrV63bWCwW/fvf/9Zjjz0md3d31a1bVx9//PENj2EYhgYPHqy6devq888/V0REhO699141a9ZMkyZN0vr1661jk5KS1LFjR7m5ualy5cp69tlnlZ2dbe3v0KGDRo8ebbP/3r17a/Dgwdb1mjVravr06Xr66adVsWJF1ahRQ//7v/9r7a9Vq5Yk6b777pPFYlGHDh1u+nsDAEofQQoAYFdz587VrFmz9MYbb+jw4cMKDw/Xo48+qhMnThQam5OToyeeeEKJiYn6/PPPVaNGDX3++ecaNGiQRo0apaNHj2rx4sWKjY3Vv/71L5ttp0yZon79+unw4cPq3r27IiMjdf78+evWlJiYqCNHjmjs2LFycCj8T6WPj48k6dKlSwoPD1elSpW0b98+rV69Wtu3b9eIESNM/w6zZs1Sy5YtdfDgQQ0fPlzDhg2zztp99dVXkqTt27crNTVVa9asMb1/AEDJIkgBAOzqjTfe0Isvvqgnn3xS9evX12uvvaZmzZrpzTfftBmXnZ2tiIgInTt3Tjt37pSfn5+k3wPSSy+9pKioKNWuXVuPPPKIpk2bpsWLF9tsP3jwYA0YMEB16tTR9OnTlZ2dbQ0of3YtxDVo0OCmta9YsUJXrlzRsmXL1LhxY3Xs2FHz58/X+++/r/T0dFO/Q/fu3TV8+HDVqVNHL774oqpUqaKdO3dKkvVcK1eurMDAQPn6+praNwCg5PGMFADAbrKysnT27Fm1bdvWpr1t27Y6dOiQTduAAQNUvXp17dixQ25ubtb2Q4cOaffu3TYzUPn5+bpy5YouX74sd3d3SVKTJk2s/R4eHvLy8lJGRsZ16zIMo0j1Hzt2TE2bNpWHh4dN7QUFBTp+/LgCAgKKtJ8/13ft1scb1QcAsD9mpAAAt4Xu3bvr8OHDSkhIsGnPzs7WlClTlJiYaF2SkpJ04sQJVahQwTrO2dnZZjuLxaKCgoLrHqtevXqSpG+//faW63ZwcCgUzPLy8gqNM1MfAMD+CFIAALvx8vJStWrVtHv3bpv23bt3KyQkxKZt2LBhmjFjhh599FGbN+Y1b95cx48fV506dQot13u+qSiaNWumkJAQzZo167ph5sKFC5Kkhg0b6tChQ7p06ZJN7Q4ODqpfv76k32/L++NbC/Pz8/XNN9+YqsfFxcW6LQCgfCBIAQDsavz48Xrttdf0wQcf6Pjx43rppZeUmJioUaNGFRo7cuRIvfrqq+rRo4e++OILSdLEiRO1bNkyTZkyRUeOHNGxY8e0atUqvfzyy8WuyWKxaMmSJfruu+/Uvn17bdq0Sd9//70OHz6sf/3rX+rVq5ckKTIyUhUqVFBUVJS++eYb7dy5UyNHjtTAgQOtt/V17NhRGzdu1MaNG/Xtt99q2LBh1iBWVP7+/nJzc9OWLVuUnp6uzMzMYp8bAKBkEKQAAHb1/PPPa8yYMRo7dqxCQ0O1ZcsWffzxx6pbt+51x48ePVpTpkxR9+7dtWfPHoWHh2vDhg3aunWr7r//frVp00Zz5sxRcHDwLdXVqlUr7d+/X3Xq1NEzzzyjhg0b6tFHH9WRI0esL8Jwd3fXp59+qvPnz+v+++/X448/rk6dOmn+/PnW/Tz99NOKiorSoEGD9NBDD6l27dp6+OGHTdXi5OSkefPmafHixapWrZo1yAEA7MdiFPWJWgAAAACAJGakAAAAAMA0ghQAAAAAmESQAgAAAACTCFIAAAAAYBJBCgAAAABMIkgBAAAAgEkEKQAAAAAwiSAFAAAAACYRpAAAAADAJIIUAAAAAJhEkAIAAAAAk/4/xzHTZtB5sXkAAAAASUVORK5CYII=",
+      "text/plain": [
+       "<Figure size 1000x600 with 1 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "# Plot the histograms\n",
+    "plt.figure(figsize=(10, 6))\n",
+    "\n",
+    "# Histogram for Input Tokens\n",
+    "plt.hist(df_token_gpt3_5['input_tokens'], bins=10, alpha=0.6, label='Input Tokens')\n",
+    "\n",
+    "# Histogram for Output Tokens\n",
+    "plt.hist(df_token_gpt3_5['output_tokens'], bins=10, alpha=0.6, label='Output Tokens')\n",
+    "\n",
+    "# Add titles and labels\n",
+    "plt.title(\"Token Summary\")\n",
+    "plt.xlabel(\"Token Count\")\n",
+    "plt.ylabel(\"Frequency\")\n",
+    "plt.legend()\n",
+    "\n",
+    "# Show the plot\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "9d81d486-bafd-454b-9a44-934ec111ad4d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Our Max Input Tokens:\t162\n",
+      "Our Max Output Tokens:\t572\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(f\"Our Max Input Tokens:\\t{max(df_token_gpt3_5.input_tokens)}\\nOur Max Output Tokens:\\t{max(df_token_gpt3_5.output_tokens)}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e6e235c3-75f4-48dd-b0cb-d7cc42426e69",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "7dea222b-a974-4ff6-9e3c-07de766b76c4",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2QAAAIjCAYAAABswtioAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABNDklEQVR4nO3deVhW1d7/8c8NyCwgKKApDjknlkMpqWVq4ZimZZkpmqeeDEtzOOZTOZaapWY5nk5Jg1Oex6ycSkltkHJIFIfMksSSwTJANAFh//7o5z7d4UjAQni/rmtfl3vttff93bfL4NPag8OyLEsAAAAAgBLnYroAAAAAACivCGQAAAAAYAiBDAAAAAAMIZABAAAAgCEEMgAAAAAwhEAGAAAAAIYQyAAAAADAEAIZAAAAABhCIAMAAAAAQwhkAIAi5XA4NGzYMNNlAABwTSCQAQDkcDiuaNmyZYvpUq9Kfn6+3n77bbVq1UqBgYGqWLGi6tevr4EDB+qrr74yXR4AAHIzXQAAwLx33nnHaf3tt9/Wxo0bC7Q3atSoJMv625588knNmzdPPXv2VP/+/eXm5qZDhw5p/fr1qlOnjlq3bm26RABAOeewLMsyXQQAoHQZNmyY5s2bp8L8iHA4HIqOjtbcuXOLobIrl5qaqqpVq+of//iH/vWvfzltsyxLJ06cUHBwsKHqzDh79qzc3d3l4sIFMgBQWvBfZADAFTl9+rRGjRqlGjVqyMPDQw0aNNDLL798RaHt+eefl4uLi1577TW7bf369WrXrp18fHxUsWJFdevWTfv373fab9CgQfL19dXPP/+sXr16ydfXV1WqVNHo0aOVl5d3yc9MTEyUZVlq06ZNgW0Oh8MpjE2cOFEOh6NAv5iYGDkcDv344492W61atdS9e3dt2bJFLVu2lJeXl8LDw+3LOVetWqXw8HB5enqqRYsW2r179wXPKSkpSd27d5evr6+uu+46zZs3T5KUkJCgDh06yMfHRzVr1tTSpUud9j958qRGjx6t8PBw+fr6ys/PT126dNGePXuc+m3ZskUOh0PLly/Xs88+q+uuu07e3t6Kj4+Xw+HQ7NmzC5zvtm3b5HA4tGzZskt+twCAokMgAwBclmVZuvvuuzV79mx17txZs2bNUoMGDTRmzBiNHDnykvs+++yzGj9+vBYtWqQnnnhC0h+XSHbr1k2+vr568cUX9dxzz+nAgQNq27atU/iRpLy8PEVGRiooKEgvv/yybr/9ds2cObPArNdf1axZU5K0cuVKnTlzpvAnfwHff/+9HnzwQfXo0UPTpk3Tb7/9ph49emjJkiV66qmn9NBDD2nSpEn64Ycf1LdvX+Xn5xc4py5duqhGjRqaMWOGatWqpWHDhikmJkadO3dWy5Yt9eKLL6pixYoaOHCgEhMT7X2PHDmi1atXq3v37po1a5bGjBmjhIQE3X777Tp+/HiBWqdMmaK1a9dq9OjRmjp1qho2bKg2bdpoyZIlBfouWbJEFStWVM+ePYv0+wIAXIIFAMBfREdHW3/+EbF69WpLkvX888879bv33nsth8Nhff/993abJCs6OtqyLMsaNWqU5eLiYsXExNjbT506ZQUEBFiPPPKI07FSUlIsf39/p/aoqChLkjV58mSnvs2aNbNatGhx2fMYOHCgJcmqVKmSdc8991gvv/yydfDgwQL9JkyYYF3oR+LixYstSVZiYqLdVrNmTUuStW3bNrvt448/tiRZXl5e1tGjR+32RYsWWZKszZs3FzinqVOn2m2//fab5eXlZTkcDmv58uV2+7fffmtJsiZMmGC3nT171srLy3OqMzEx0fLw8HD6njZv3mxJsurUqWOdOXPGqf/5uv78XeTk5FiVK1e2oqKiCnwPAIDiwwwZAOCy1q1bJ1dXVz355JNO7aNGjZJlWVq/fr1Tu2VZGjZsmObMmaN3331XUVFR9raNGzcqPT1d/fr10y+//GIvrq6uatWqlTZv3lzg8x977DGn9Xbt2unIkSOXrXvx4sWaO3euateurffff1+jR49Wo0aN1LFjR/38889X8xU4ady4sSIiIuz1Vq1aSZI6dOigsLCwAu0XqvUf//iH/eeAgAA1aNBAPj4+6tu3r93eoEEDBQQEOO3v4eFh3wOWl5enX3/9Vb6+vmrQoIG++eabAp8TFRUlLy8vp7a+ffvK09PTaZbs448/1i+//KKHHnroyr4EAECRIJABAC7r6NGjqlatmipWrOjUfv6pi0ePHnVqf/vttzVv3jy99tpr6tevn9O2w4cPS/ojvFSpUsVp+eSTT5SWlubU39PTU1WqVHFqq1Spkn777bfL1u3i4qLo6Gjt2rVLv/zyiz744AN16dJFn376qR544IErO/kL+HPokiR/f39JUo0aNS7Y/tdaL3RO/v7+ql69eoF72fz9/Z32z8/P1+zZs1WvXj15eHiocuXKqlKlivbu3auMjIwCtdauXbtAW0BAgHr06OF0f9qSJUt03XXXqUOHDhc9bwBA0eOx9wCAItemTRvFx8dr7ty56tu3rwIDA+1t5++neueddxQaGlpgXzc35x9Nrq6uRVJTUFCQ7r77bt19991q3769tm7dqqNHj6pmzZoXfKCHpIs+OORiNV2s3frLg0/+zv5Tp07Vc889p4cfflhTpkxRYGCgXFxcNGLEiAL3qkkqMDt23sCBA7Vy5Upt27ZN4eHh+vDDD/X444/zBEYAKGEEMgDAZdWsWVObNm3SqVOnnGbJvv32W3v7n9WtW1czZsxQ+/bt1blzZ8XGxtr7XX/99ZKk4OBgderUqYTOwFnLli21detWJScnq2bNmqpUqZIkKT09XQEBAXa/v878lQb/+c9/dMcdd+iNN95wak9PT1flypWv+DidO3dWlSpVtGTJErVq1UpnzpzRgAEDirpcAMBl8L/BAACX1bVrV+Xl5RV4t9js2bPlcDjUpUuXAvs0bdpU69at08GDB9WjRw/9/vvvkqTIyEj5+flp6tSpys3NLbDfiRMniqTmlJQUHThwoEB7Tk6OYmNj5eLiorp160r6b0j87LPP7H6nT5/WW2+9VSS1FCVXV9cCM24rV6686nvi3Nzc1K9fP7333nuKiYlReHi4mjZtWpSlAgCuADNkAIDL6tGjh+644w4988wz+vHHH3XjjTfqk08+0QcffKARI0bYgeavWrdurQ8++EBdu3bVvffeq9WrV8vPz08LFizQgAED1Lx5cz3wwAOqUqWKkpKStHbtWrVp06ZIXir9008/6ZZbblGHDh3UsWNHhYaGKi0tTcuWLdOePXs0YsQIe0bprrvuUlhYmIYMGaIxY8bI1dVVb775pl1XadK9e3dNnjxZgwcP1q233qqEhAQtWbJEderUuepjDRw4UK+++qo2b96sF198sRiqBQBcDoEMAHBZLi4u+vDDDzV+/HitWLFCixcvVq1atfTSSy9p1KhRl9y3Q4cOeu+999SnTx8NGDBAS5cu1YMPPqhq1app+vTpeumll5Sdna3rrrtO7dq10+DBg4uk5gYNGuiVV17RunXrNH/+fKWmpsrT01NNmjTR66+/riFDhth9K1SooPfff1+PP/64nnvuOYWGhmrEiBGqVKlSkdVTVP73f/9Xp0+f1tKlS7VixQo1b95ca9eu1dNPP33Vx2rRooVuuOEGHTx4UP379y+GagEAl+Ow/nrdAwAAKDeaNWumwMBAxcbGmi4FAMol7iEDAKCc2rlzp+Lj4zVw4EDTpQBAucUMGQAA5cy+ffu0a9cuzZw5U7/88ouOHDkiT09P02UBQLnEDBkAAOXMf/7zHw0ePFi5ublatmwZYQwADGKGDAAAAAAMYYYMAAAAAAwhkAEAAACAIbyHTFJ+fr6OHz+uihUryuFwmC4HAAAAgCGWZenUqVOqVq2aXFyKf/6KQCbp+PHjqlGjhukyAAAAAJQSx44dU/Xq1Yv9cwhkkipWrCjpjy/dz8/PcDUAAAAATMnMzFSNGjXsjFDcCGSSfZmin58fgQwAAABAid3KxEM9AAAAAMAQAhkAAAAAGEIgAwAAAABDuIcMAAAApZ5lWTp37pzy8vJMl4JrnKurq9zc3ErN664IZAAAACjVcnJylJycrDNnzpguBWWEt7e3qlatKnd3d9OlEMgAAABQeuXn5ysxMVGurq6qVq2a3N3dS83MBq49lmUpJydHJ06cUGJiourVq1ciL3++FAIZAAAASq2cnBzl5+erRo0a8vb2Nl0OygAvLy9VqFBBR48eVU5Ojjw9PY3Ww0M9AAAAUOqZnsVA2VKaxlPpqQQAAAAAyhkCGQAAAAAYwj1kAAAAuCaNW5VQop83rXd4iX5eaVarVi2NGDFCI0aMMF3KNY8ZMgAAAKAYDBo0SL169Srxz42JiVFAQMAl+7Rv314Oh+OiS/v27UukVjBDBgAAAJQ7q1atUk5OjiTp2LFjuuWWW7Rp0ybdcMMNklQq3s9VXjBDBgAAAJSA9u3b68knn9Q///lPBQYGKjQ0VBMnTnTq43A4tGDBAnXp0kVeXl6qU6eO/vOf/9jbt2zZIofDofT0dLstPj5eDodDP/74o7Zs2aLBgwcrIyPDnu3662dIsj8/NDRUVapUkSQFBQXZbZs3b9YNN9wgDw8P1apVSzNnzrzkuf373/9WQECAYmNjJUn79u1Tly5d5Ovrq5CQEA0YMEC//PLLFX8XlmVp4sSJCgsLk4eHh6pVq6Ynn3zyCr/pawuBDAAAACghb731lnx8fPT1119rxowZmjx5sjZu3OjU57nnnlOfPn20Z88e9e/fXw888IAOHjx4Rce/9dZb9corr8jPz0/JyclKTk7W6NGjr6rGXbt2qW/fvnrggQeUkJCgiRMn6rnnnlNMTMwF+8+YMUNPP/20PvnkE3Xs2FHp6enq0KGDmjVrpp07d2rDhg1KTU1V3759r/i7+L//+z/Nnj1bixYt0uHDh7V69WqFh5fNe/i4ZBEAAAAoIU2bNtWECRMkSfXq1dPcuXMVGxurO++80+5z33336R//+IckacqUKdq4caNee+01zZ8//7LHd3d3l7+/vxwOh0JDQwtV46xZs9SxY0c999xzkqT69evrwIEDeumllzRo0CCnvmPHjtU777yjrVu32pc7zp07V82aNdPUqVPtfm+++aZq1Kih7777TvXr17/sd5GUlKTQ0FB16tRJFSpUUFhYmG655ZZCnU9pxwwZAAAAUEKaNm3qtF61alWlpaU5tUVERBRYv9IZsqJw8OBBtWnTxqmtTZs2Onz4sPLy8uy2mTNn6vXXX9cXX3xhhzFJ2rNnjzZv3ixfX197adiwoSTphx9+sPtd6ru477779Pvvv6tOnTp65JFH9P777+vcuXNFfq6lAYEMAAAAKCEVKlRwWnc4HMrPz7/i/V1c/vj13bIsuy03N7doirtK7dq1U15ent577z2n9qysLPXo0UPx8fFOy+HDh3XbbbfZ/S71XdSoUUOHDh3S/Pnz5eXlpccff1y33XabsXMtTgQyAAAAoBT56quvCqw3atRIkuwHcCQnJ9vb4+Pjnfq7u7s7zWRdrUaNGunLL790avvyyy9Vv359ubq62m233HKL1q9fr6lTp+rll1+225s3b679+/erVq1aqlu3rtPi4+NzxXV4eXmpR48eevXVV7VlyxbFxcUpIaFk3z1XEriH7Br015cg8pJCAACAsmPlypVq2bKl2rZtqyVLlmj79u164403JEl169ZVjRo1NHHiRL3wwgv67rvvCjwBsVatWsrKylJsbKxuvPFGeXt7y9vb+4o/f9SoUbr55ps1ZcoU3X///YqLi9PcuXMveA/brbfeqnXr1qlLly5yc3PTiBEjFB0drddff139+vWzn6L4/fffa/ny5fr3v//tFOouJiYmRnl5eWrVqpW8vb317rvvysvLSzVr1rzi87hWEMgAAABwTSqr/1N60qRJWr58uR5//HFVrVpVy5YtU+PGjSX9cZnfsmXLNHToUDVt2lQ333yznn/+ed133332/rfeeqsee+wx3X///fr11181YcKECz76/mKaN2+u9957T+PHj9eUKVNUtWpVTZ48ucADPc5r27at1q5dq65du8rV1VVPPPGEvvzyS40dO1Z33XWXsrOzVbNmTXXu3Nm+5PJyAgICNH36dI0cOVJ5eXkKDw/XRx99pKCgoCs+j2uFw/rzBajlVGZmpvz9/ZWRkSE/Pz/T5VwWM2QAAKC8OHv2rBITE1W7dm15enqaLqfYORwOvf/+++rVq5fpUsq0S42rks4G3EMGAAAAAIYQyAAAAADAEO4hAwAAAEoJ7iYqf5ghAwAAAABDCGQAAAAAYAiXLAIomz4abroCM3rMMV0BAAC4CkZnyCZOnCiHw+G0NGzY0N5+9uxZRUdHKygoSL6+vurTp49SU1OdjpGUlKRu3brJ29tbwcHBGjNmjM6dO1fSpwIAAAAAV834DNkNN9ygTZs22etubv8t6amnntLatWu1cuVK+fv7a9iwYerdu7e+/PJLSVJeXp66deum0NBQbdu2TcnJyRo4cKAqVKigqVOnlvi5AAAAAMDVMB7I3NzcFBoaWqA9IyNDb7zxhpYuXaoOHTpIkhYvXqxGjRrpq6++UuvWrfXJJ5/owIED2rRpk0JCQnTTTTdpypQpGjt2rCZOnCh3d/eSPh0AAAAAuGLGA9nhw4dVrVo1eXp6KiIiQtOmTVNYWJh27dql3NxcderUye7bsGFDhYWFKS4uTq1bt1ZcXJzCw8MVEhJi94mMjNTQoUO1f/9+NWvW7IKfmZ2drezsbHs9MzOz+E4QAAAAxaOk7xfmPt1CmzhxolavXq34+HjTpZQ6Ru8ha9WqlWJiYrRhwwYtWLBAiYmJateunU6dOqWUlBS5u7srICDAaZ+QkBClpKRIklJSUpzC2Pnt57ddzLRp0+Tv728vNWrUKNoTAwAAACQdO3ZMDz/8sKpVqyZ3d3fVrFlTw4cP16+//npVx/nxxx/lcDiKLdA4HA6tXr36ottjYmIKPPvhr8uPP/5YLLWVdUYDWZcuXXTfffepadOmioyM1Lp165Senq733nuvWD933LhxysjIsJdjx44V6+cBAACg/Dly5Ihatmypw4cPa9myZfr++++1cOFCxcbGKiIiQidPnjRd4hW7//77lZycbC8RERF65JFHnNqY5CicUvUesoCAANWvX1/ff/+9QkNDlZOTo/T0dKc+qamp9j1noaGhBZ66eH79Qvelnefh4SE/Pz+nBQAAAChK0dHRcnd31yeffKLbb79dYWFh6tKlizZt2qSff/5ZzzzzjN33QjNUAQEBiomJkSTVrl1bktSsWTM5HA61b99ekjRo0CD16tVLkyZNUpUqVeTn56fHHntMOTk59nFq1aqlV155xenYN910kyZOnGhvl6R77rlHDofDXv8zLy8vhYaG2ou7u7u8vb3t9ZycHPXu3Vu+vr7y8/NT3759C/ye/mc//PCD6tSpo2HDhsmyLGVnZ2v06NG67rrr5OPjo1atWmnLli12/5iYGAUEBOjjjz9Wo0aN5Ovrq86dOys5Odnus2XLFt1yyy3y8fFRQECA2rRpo6NHj160htKiVAWyrKws/fDDD6patapatGihChUqKDY21t5+6NAhJSUlKSIiQpIUERGhhIQEpaWl2X02btwoPz8/NW7cuMTrBwAAACTp5MmT+vjjj/X444/Ly8vLaVtoaKj69++vFStWyLKsKzre9u3bJUmbNm1ScnKyVq1aZW+LjY3VwYMHtWXLFi1btkyrVq3SpEmTrrjWHTt2SPrjAXrJycn2+pXKz89Xz549dfLkSW3dulUbN27UkSNHdP/991+w/969e9W2bVs9+OCDmjt3rhwOh4YNG6a4uDgtX75ce/fu1X333afOnTvr8OHD9n5nzpzRyy+/rHfeeUefffaZkpKSNHr0aEnSuXPn1KtXL91+++3au3ev4uLi9Oijj8rhcFzVuZhg9KEeo0ePVo8ePVSzZk0dP35cEyZMkKurq/r16yd/f38NGTJEI0eOVGBgoPz8/PTEE08oIiJCrVu3liTdddddaty4sQYMGKAZM2YoJSVFzz77rKKjo+Xh4WHy1AAAAFCOHT58WJZlqVGjRhfc3qhRI/322286ceKEgoODL3u8KlWqSJKCgoIKXAnm7u6uN998U97e3rrhhhs0efJkjRkzRlOmTJGLy+XnX84fOyAg4JJXmV1MbGysEhISlJiYaF+2+Pbbb+uGG27Qjh07dPPNN9t9t23bpu7du+uZZ57RqFGjJP3xXuHFixcrKSlJ1apVk/RHTtiwYYMWL15sv84qNzdXCxcu1PXXXy9JGjZsmCZPnizpj4f0ZWRkqHv37vb2i333pY3RQPbTTz+pX79++vXXX1WlShW1bdtWX331lT0oZs+eLRcXF/Xp00fZ2dmKjIzU/Pnz7f1dXV21Zs0aDR06VBEREfLx8VFUVJT9FwMAAACYdKUzYH/HjTfeKG9vb3s9IiJCWVlZOnbsmGrWrFnsn3/w4EHVqFHD6R6yxo0bKyAgQAcPHrQDWVJSku6880698MILGjFihN03ISFBeXl5ql+/vtNxs7OzFRQUZK97e3vbYUuSqlatal8pFxgYqEGDBikyMlJ33nmnOnXqpL59+6pq1arFccpFymggW758+SW3e3p6at68eZo3b95F+9SsWVPr1q0r6tIAAACAQqtbt64cDocOHjyoe+65p8D2gwcPqlKlSvZEhMPhKBDecnNzi6QWFxeXYjv21ahSpYqqVaumZcuW6eGHH7af45CVlSVXV1ft2rVLrq6uTvv4+vraf65QoYLTtr9+Z4sXL9aTTz6pDRs2aMWKFXr22We1ceNG++q60qpU3UMGAAAAlAVBQUG68847NX/+fP3+++9O21JSUrRkyRLdf//99j1OVapUcXpAxeHDh3XmzBl73d3dXZKUl5dX4LP27Nnj9BlfffWVfH197Rmrvx47MzNTiYmJTseoUKHCBY99JRo1aqRjx445Pbn8wIEDSk9Pd3qug5eXl9asWSNPT09FRkbq1KlTkv54UEleXp7S0tJUt25dp+VqL6Fs1qyZxo0bp23btqlJkyZaunRpoc6pJBHIAAAAgGIwd+5c+7abzz77TMeOHdOGDRt055136rrrrtMLL7xg9+3QoYPmzp2r3bt3a+fOnXrsscecZoSCg4Pl5eWlDRs2KDU1VRkZGfa2nJwcDRkyRAcOHNC6des0YcIEDRs2zL5/rEOHDnrnnXf0+eefKyEhQVFRUQVmomrVqqXY2FilpKTot99+u6rz7NSpk8LDw9W/f39988032r59uwYOHKjbb79dLVu2dOrr4+OjtWvXys3NTV26dFFWVpbq16+v/v37a+DAgVq1apUSExO1fft2TZs2TWvXrr2iGhITEzVu3DjFxcXp6NGj+uSTT3T48OFr4j4yo5csAgAAAIXWY47pCi6pXr162rlzpyZMmKC+ffvq5MmTCg0NVa9evTRhwgQFBgbafWfOnKnBgwerXbt2qlatmubMmaNdu3bZ293c3PTqq69q8uTJGj9+vNq1a2c/Fr5jx46qV6+ebrvtNmVnZ6tfv372I+2lP97Bm5iYqO7du8vf319TpkwpMEM2c+ZMjRw5Uq+//rquu+66q3rJs8Ph0AcffKAnnnhCt912m1xcXNS5c2e99tprF+zv6+ur9evXKzIyUt26ddO6deu0ePFiPf/88xo1apR+/vlnVa5cWa1bt1b37t2vqAZvb299++23euutt/Trr7+qatWqio6O1v/8z/9c8XmY4rBK4k7DUi4zM1P+/v7KyMi4Jt5JNm5VgtP6tN7hhioBSrGPhpuuwIxS/ssJAFyts2fPKjExUbVr15anp6fpckqdQYMGKT09vcA7zHBplxpXJZ0NuGQRAAAAAAwhkAEAAACAIdxDBgAAAFyjYmJiTJeAv4kZMgAAAAAwhEAGAACAUo/n0KEolabxRCADAABAqXX+XVx/fkky8HedH09/ftebKdxDBgAAgFLL1dVVAQEBSktLk/TH+6YcDofhqnCtsixLZ86cUVpamgICAgq8INsEAhkAAABKtdDQUEmyQxnwdwUEBNjjyjQCGQAAAEo1h8OhqlWrKjg4WLm5uabLwTWuQoUKpWJm7DwCGQAAAK4Jrq6upeoXaaAo8FAPAAAAADCEQAYAAAAAhhDIAAAAAMAQAhkAAAAAGEIgAwAAAABDCGQAAAAAYAiBDAAAAAAMIZABAAAAgCEEMgAAAAAwhEAGAAAAAIYQyAAAAADAEAIZAAAAABhCIAMAAAAAQwhkAAAAAGAIgQwAAAAADCGQAQAAAIAhBDIAAAAAMIRABgAAAACGEMgAAAAAwBACGQAAAAAYQiADAAAAAEMIZAAAAABgCIEMAAAAAAwhkAEAAACAIQQyAAAAADCEQAYAAAAAhhDIAAAAAMAQAhkAAAAAGEIgAwAAAABDCGQAAAAAYAiBDAAAAAAMIZABAAAAgCEEMgAAAAAwhEAGAAAAAIYQyAAAAADAEAIZAAAAABhCIAMAAAAAQwhkAAAAAGAIgQwAAAAADCGQAQAAAIAhbqYLAAAUoY+Gm67AjB5zTFcAAEChMEMGAAAAAIYQyAAAAADAEAIZAAAAABhCIAMAAAAAQwhkAAAAAGAIgQwAAAAADCGQAQAAAIAhBDIAAAAAMIRABgAAAACGEMgAAAAAwBACGQAAAAAYQiADAAAAAEMIZAAAAABgCIEMAAAAAAwhkAEAAACAIQQyAAAAADCEQAYAAAAAhhDIAAAAAMAQAhkAAAAAGEIgAwAAAABDSk0gmz59uhwOh0aMGGG3nT17VtHR0QoKCpKvr6/69Omj1NRUp/2SkpLUrVs3eXt7Kzg4WGPGjNG5c+dKuHoAAAAAuHqlIpDt2LFDixYtUtOmTZ3an3rqKX300UdauXKltm7dquPHj6t379729ry8PHXr1k05OTnatm2b3nrrLcXExGj8+PElfQoAAAAAcNWMB7KsrCz1799fr7/+uipVqmS3Z2Rk6I033tCsWbPUoUMHtWjRQosXL9a2bdv01VdfSZI++eQTHThwQO+++65uuukmdenSRVOmTNG8efOUk5Nj6pQAAAAA4IoYD2TR0dHq1q2bOnXq5NS+a9cu5ebmOrU3bNhQYWFhiouLkyTFxcUpPDxcISEhdp/IyEhlZmZq//79F/3M7OxsZWZmOi0AAAAAUNLcTH748uXL9c0332jHjh0FtqWkpMjd3V0BAQFO7SEhIUpJSbH7/DmMnd9+ftvFTJs2TZMmTfqb1QMAAADA32NshuzYsWMaPny4lixZIk9PzxL97HHjxikjI8Nejh07VqKfDwAAAACSwUC2a9cupaWlqXnz5nJzc5Obm5u2bt2qV199VW5ubgoJCVFOTo7S09Od9ktNTVVoaKgkKTQ0tMBTF8+vn+9zIR4eHvLz83NaAAAAAKCkGQtkHTt2VEJCguLj4+2lZcuW6t+/v/3nChUqKDY21t7n0KFDSkpKUkREhCQpIiJCCQkJSktLs/ts3LhRfn5+aty4cYmfEwAAAABcDWP3kFWsWFFNmjRxavPx8VFQUJDdPmTIEI0cOVKBgYHy8/PTE088oYiICLVu3VqSdNddd6lx48YaMGCAZsyYoZSUFD377LOKjo6Wh4dHiZ8TAAAAAFwNow/1uJzZs2fLxcVFffr0UXZ2tiIjIzV//nx7u6urq9asWaOhQ4cqIiJCPj4+ioqK0uTJkw1WDQAAAABXxmFZlmW6CNMyMzPl7++vjIyMa+J+snGrEi65fVrv8BKqBCjFPhpuugKUpB5zTFcAACgjSjobGH8PGQAAAACUVwQyAAAAADCEQAYAAAAAhhDIAAAAAMAQAhkAAAAAGEIgAwAAAABDCGQAAAAAYAiBDAAAAAAMIZABAAAAgCEEMgAAAAAwhEAGAAAAAIYQyAAAAADAEAIZAAAAABhCIAMAAAAAQwhkAAAAAGAIgQwAAAAADCGQAQAAAIAhBDIAAAAAMIRABgAAAACGEMgAAAAAwBACGQAAAAAYQiADAAAAAEMIZAAAAABgCIEMAAAAAAwhkAEAAACAIQQyAAAAADCEQAYAAAAAhhDIAAAAAMAQAhkAAAAAGEIgAwAAAABDCGQAAAAAYAiBDAAAAAAMIZABAAAAgCEEMgAAAAAwhEAGAAAAAIYQyAAAAADAEAIZAAAAABhCIAMAAAAAQwhkAAAAAGAIgQwAAAAADCGQAQAAAIAhBDIAAAAAMIRABgAAAACGEMgAAAAAwBACGQAAAAAYQiADAAAAAEMIZAAAAABgCIEMAAAAAAwhkAEAAACAIQQyAAAAADCEQAYAAAAAhhDIAAAAAMAQAhkAAAAAGEIgAwAAAABDCGQAAAAAYAiBDAAAAAAMIZABAAAAgCEEMgAAAAAwhEAGAAAAAIYQyAAAAADAEAIZAAAAABhCIAMAAAAAQwhkAAAAAGAIgQwAAAAADCGQAQAAAIAhbqYLwOWNW5VgugQAAAAAxYAZMgAAAAAwhEAGAAAAAIYQyAAAAADAEAIZAAAAABhSqEB25MiRoq4DAAAAAMqdQgWyunXr6o477tC7776rs2fPFnVNAAAAAFAuFCqQffPNN2ratKlGjhyp0NBQ/c///I+2b99e1LUBAAAAQJlWqEB20003ac6cOTp+/LjefPNNJScnq23btmrSpIlmzZqlEydOFHWdAAAAAFDm/K2Heri5ual3795auXKlXnzxRX3//fcaPXq0atSooYEDByo5OfmS+y9YsEBNmzaVn5+f/Pz8FBERofXr19vbz549q+joaAUFBcnX11d9+vRRamqq0zGSkpLUrVs3eXt7Kzg4WGPGjNG5c+f+zmkBAAAAQIn4W4Fs586devzxx1W1alXNmjVLo0eP1g8//KCNGzfq+PHj6tmz5yX3r169uqZPn65du3Zp586d6tChg3r27Kn9+/dLkp566il99NFHWrlypbZu3arjx4+rd+/e9v55eXnq1q2bcnJytG3bNr311luKiYnR+PHj/85pAQAAAECJcFiWZV3tTrNmzdLixYt16NAhde3aVf/4xz/UtWtXubj8N9/99NNPqlWr1lXPVgUGBuqll17SvffeqypVqmjp0qW69957JUnffvutGjVqpLi4OLVu3Vrr169X9+7ddfz4cYWEhEiSFi5cqLFjx+rEiRNyd3e/os/MzMyUv7+/MjIy5Ofnd1X1loRxqxKuqv+03uHFVAlwDflouOkKUJJ6zDFdAQCgjCjpbFCoGbIFCxbowQcf1NGjR7V69Wp1797dKYxJUnBwsN54440rPmZeXp6WL1+u06dPKyIiQrt27VJubq46depk92nYsKHCwsIUFxcnSYqLi1N4eLgdxiQpMjJSmZmZ9izbhWRnZyszM9NpAQAAAICS5laYnQ4fPnzZPu7u7oqKirpsv4SEBEVEROjs2bPy9fXV+++/r8aNGys+Pl7u7u4KCAhw6h8SEqKUlBRJUkpKilMYO7/9/LaLmTZtmiZNmnTZ2gAAAACgOBVqhmzx4sVauXJlgfaVK1fqrbfeuqpjNWjQQPHx8fr66681dOhQRUVF6cCBA4Up64qNGzdOGRkZ9nLs2LFi/TwAAAAAuJBCzZBNmzZNixYtKtAeHBysRx999Ipmxs5zd3dX3bp1JUktWrTQjh07NGfOHN1///3KyclRenq60yxZamqqQkNDJUmhoaEF3n92/imM5/tciIeHhzw8PK64RuCax/1UAAAApVKhZsiSkpJUu3btAu01a9ZUUlLS3yooPz9f2dnZatGihSpUqKDY2Fh726FDh5SUlKSIiAhJUkREhBISEpSWlmb32bhxo/z8/NS4ceO/VQcAAAAAFLdCzZAFBwdr7969qlWrllP7nj17FBQUdMXHGTdunLp06aKwsDCdOnVKS5cu1ZYtW/Txxx/L399fQ4YM0ciRIxUYGCg/Pz898cQTioiIUOvWrSVJd911lxo3bqwBAwZoxowZSklJ0bPPPqvo6GhmwAAAAACUeoUKZP369dOTTz6pihUr6rbbbpMkbd26VcOHD9cDDzxwxcdJS0uzXyDt7++vpk2b6uOPP9add94pSZo9e7ZcXFzUp08fZWdnKzIyUvPnz7f3d3V11Zo1azR06FBFRETIx8dHUVFRmjx5cmFOCwAAAABKVKHeQ5aTk6MBAwZo5cqVcnP7I9Pl5+dr4MCBWrhw4RW//6u04D1kKPO4hwxlHe8hAwAUkZLOBoWaIXN3d9eKFSs0ZcoU7dmzR15eXgoPD1fNmjWLuj4AAAAAKLMKFcjOq1+/vurXr19UtQAAAABAuVKoQJaXl6eYmBjFxsYqLS1N+fn5Tts//fTTIikOAAAAAMqyQgWy4cOHKyYmRt26dVOTJk3kcDiKui4AAAAAKPMKFciWL1+u9957T127di3qegAAAACg3CjUi6Hd3d1Vt27doq4FAAAAAMqVQgWyUaNGac6cOSrEE/MBAAAAAP9foS5Z/OKLL7R582atX79eN9xwgypUqOC0fdWqVUVSHAAAAACUZYUKZAEBAbrnnnuKuhYAAAAAKFcKFcgWL15c1HUAAAAAQLlTqHvIJOncuXPatGmTFi1apFOnTkmSjh8/rqysrCIrDgAAAADKskLNkB09elSdO3dWUlKSsrOzdeedd6pixYp68cUXlZ2drYULFxZ1nQAAAABQ5hRqhmz48OFq2bKlfvvtN3l5ednt99xzj2JjY4usOAAAAAAoywo1Q/b5559r27Ztcnd3d2qvVauWfv755yIpDAAAAADKukLNkOXn5ysvL69A+08//aSKFSv+7aIAAAAAoDwoVCC766679Morr9jrDodDWVlZmjBhgrp27VpUtQEAAABAmVaoSxZnzpypyMhINW7cWGfPntWDDz6ow4cPq3Llylq2bFlR1wgAAAAAZVKhAln16tW1Z88eLV++XHv37lVWVpaGDBmi/v37Oz3kAwAAAABwcYUKZJLk5uamhx56qChrAQAAAIBypVCB7O23377k9oEDBxaqGAAAAAAoTwoVyIYPH+60npubqzNnzsjd3V3e3t4EMgAAAAC4AoV6yuJvv/3mtGRlZenQoUNq27YtD/UAAAAAgCtUqEB2IfXq1dP06dMLzJ4BAAAAAC6syAKZ9MeDPo4fP16UhwQAAACAMqtQ95B9+OGHTuuWZSk5OVlz585VmzZtiqQwAAAAACjrChXIevXq5bTucDhUpUoVdejQQTNnziyKugAAAACgzCtUIMvPzy/qOgAAAACg3CnSe8gAAAAAAFeuUDNkI0eOvOK+s2bNKsxHAAAAAECZV6hAtnv3bu3evVu5ublq0KCBJOm7776Tq6urmjdvbvdzOBxFUyUAAAAAlEGFCmQ9evRQxYoV9dZbb6lSpUqS/nhZ9ODBg9WuXTuNGjWqSIsEAAAAgLKoUPeQzZw5U9OmTbPDmCRVqlRJzz//PE9ZBAAAAIArVKhAlpmZqRMnThRoP3HihE6dOvW3iwIAAACA8qBQgeyee+7R4MGDtWrVKv3000/66aef9H//938aMmSIevfuXdQ1AgAAAECZVKh7yBYuXKjRo0frwQcfVG5u7h8HcnPTkCFD9NJLLxVpgQAAAABQVhUqkHl7e2v+/Pl66aWX9MMPP0iSrr/+evn4+BRpcQAAAABQlv2tF0MnJycrOTlZ9erVk4+PjyzLKqq6AAAAAKDMK1Qg+/XXX9WxY0fVr19fXbt2VXJysiRpyJAhPPIeAAAAAK5QoQLZU089pQoVKigpKUne3t52+/33368NGzYUWXEAAAAAUJYV6h6yTz75RB9//LGqV6/u1F6vXj0dPXq0SAoDAAAAgLKuUDNkp0+fdpoZO+/kyZPy8PD420UBAAAAQHlQqEDWrl07vf322/a6w+FQfn6+ZsyYoTvuuKPIigMAAACAsqxQlyzOmDFDHTt21M6dO5WTk6N//vOf2r9/v06ePKkvv/yyqGsEAODSPhpuugIzeswxXQEA4G8q1AxZkyZN9N1336lt27bq2bOnTp8+rd69e2v37t26/vrri7pGAAAAACiTrnqGLDc3V507d9bChQv1zDPPFEdNAAAAAFAuXPUMWYUKFbR3797iqAUAAAAAypVCXbL40EMP6Y033ijqWgAAAACgXCnUQz3OnTunN998U5s2bVKLFi3k4+PjtH3WrFlFUhwAAAAAlGVXFciOHDmiWrVqad++fWrevLkk6bvvvnPq43A4iq46AAAAACjDriqQ1atXT8nJydq8ebMk6f7779err76qkJCQYikOAAAAAMqyq7qHzLIsp/X169fr9OnTRVoQAAAAAJQXhXqox3l/DWgAAAAAgCt3VYHM4XAUuEeMe8YAAAAAoHCu6h4yy7I0aNAgeXh4SJLOnj2rxx57rMBTFletWlV0FQIAAABAGXVVgSwqKspp/aGHHirSYgAAAACgPLmqQLZ48eLiqgMAAAAAyp2/9VAPAAAAAEDhEcgAAAAAwBACGQAAAAAYQiADAAAAAEMIZAAAAABgCIEMAAAAAAwhkAEAAACAIQQyAAAAADCEQAYAAAAAhhDIAAAAAMAQAhkAAAAAGEIgAwAAAABDCGQAAAAAYAiBDAAAAAAMIZABAAAAgCEEMgAAAAAwhEAGAAAAAIYQyAAAAADAEKOBbNq0abr55ptVsWJFBQcHq1evXjp06JBTn7Nnzyo6OlpBQUHy9fVVnz59lJqa6tQnKSlJ3bp1k7e3t4KDgzVmzBidO3euJE8FAAAAAK6a0UC2detWRUdH66uvvtLGjRuVm5uru+66S6dPn7b7PPXUU/roo4+0cuVKbd26VcePH1fv3r3t7Xl5eerWrZtycnK0bds2vfXWW4qJidH48eNNnBIAAAAAXDGHZVmW6SLOO3HihIKDg7V161bddtttysjIUJUqVbR06VLde++9kqRvv/1WjRo1UlxcnFq3bq3169ere/fuOn78uEJCQiRJCxcu1NixY3XixAm5u7tf9nMzMzPl7++vjIwM+fn5Fes5Fsa4VQlX1X9a7/BiqgTXrI+Gm64AQHHoMcd0BQBQ5pR0NihV95BlZGRIkgIDAyVJu3btUm5urjp16mT3adiwocLCwhQXFydJiouLU3h4uB3GJCkyMlKZmZnav3//BT8nOztbmZmZTgsAAAAAlLRSE8jy8/M1YsQItWnTRk2aNJEkpaSkyN3dXQEBAU59Q0JClJKSYvf5cxg7v/38tguZNm2a/P397aVGjRpFfDYAAAAAcHmlJpBFR0dr3759Wr58ebF/1rhx45SRkWEvx44dK/bPBAAAAIC/cjNdgCQNGzZMa9as0Weffabq1avb7aGhocrJyVF6errTLFlqaqpCQ0PtPtu3b3c63vmnMJ7v81ceHh7y8PAo4rMAAAAAgKtjdIbMsiwNGzZM77//vj799FPVrl3baXuLFi1UoUIFxcbG2m2HDh1SUlKSIiIiJEkRERFKSEhQWlqa3Wfjxo3y8/NT48aNS+ZEAAAAAKAQjM6QRUdHa+nSpfrggw9UsWJF+54vf39/eXl5yd/fX0OGDNHIkSMVGBgoPz8/PfHEE4qIiFDr1q0lSXfddZcaN26sAQMGaMaMGUpJSdGzzz6r6OhoZsEAAAAAlGpGA9mCBQskSe3bt3dqX7x4sQYNGiRJmj17tlxcXNSnTx9lZ2crMjJS8+fPt/u6urpqzZo1Gjp0qCIiIuTj46OoqChNnjy5pE4DAAAAAArFaCC7klegeXp6at68eZo3b95F+9SsWVPr1q0rytIAAAAAoNiVmqcsAgAAAEB5QyADAAAAAEMIZAAAAABgSKl4DxmK1rhVCfafp/UON1gJAAAAgEthhgwAAAAADCGQAQAAAIAhBDIAAAAAMIRABgAAAACGEMgAAAAAwBACGQAAAAAYQiADAAAAAEMIZAAAAABgCIEMAAAAAAwhkAEAAACAIQQyAAAAADCEQAYAAAAAhhDIAAAAAMAQAhkAAAAAGEIgAwAAAABDCGQAAAAAYAiBDAAAAAAMIZABAAAAgCEEMgAAAAAwhEAGAAAAAIYQyAAAAADAEAIZAAAAABhCIAMAAAAAQwhkAAAAAGAIgQwAAAAADCGQAQAAAIAhBDIAAAAAMIRABgAAAACGEMgAAAAAwBACGQAAAAAYQiADAAAAAEMIZAAAAABgCIEMAAAAAAwhkAEAAACAIQQyAAAAADCEQAYAAAAAhhDIAAAAAMAQAhkAAAAAGEIgAwAAAABDCGQAAAAAYAiBDAAAAAAMIZABAAAAgCEEMgAAAAAwhEAGAAAAAIYQyAAAAADAEAIZAAAAABhCIAMAAAAAQwhkAAAAAGAIgQwAAAAADCGQAQAAAIAhBDIAAAAAMIRABgAAAACGEMgAAAAAwBACGQAAAAAYQiADAAAAAEMIZAAAAABgCIEMAAAAAAxxM10Aite4VQlO69N6hxuqBAAAAMBfMUMGAAAAAIYQyAAAAADAEAIZAAAAABhCIAMAAAAAQwhkAAAAAGAIgQwAAAAADCGQAQAAAIAhBDIAAAAAMIRABgAAAACGEMgAAAAAwBACGQAAAAAYQiADAAAAAEMIZAAAAABgiNFA9tlnn6lHjx6qVq2aHA6HVq9e7bTdsiyNHz9eVatWlZeXlzp16qTDhw879Tl58qT69+8vPz8/BQQEaMiQIcrKyirBswAAAACAwjEayE6fPq0bb7xR8+bNu+D2GTNm6NVXX9XChQv19ddfy8fHR5GRkTp79qzdp3///tq/f782btyoNWvW6LPPPtOjjz5aUqcAAAAAAIXmZvLDu3Tpoi5dulxwm2VZeuWVV/Tss8+qZ8+ekqS3335bISEhWr16tR544AEdPHhQGzZs0I4dO9SyZUtJ0muvvaauXbvq5ZdfVrVq1S547OzsbGVnZ9vrmZmZRXxmAAAAAHB5pfYessTERKWkpKhTp052m7+/v1q1aqW4uDhJUlxcnAICAuwwJkmdOnWSi4uLvv7664see9q0afL397eXGjVqFN+JAAAAAMBFlNpAlpKSIkkKCQlxag8JCbG3paSkKDg42Gm7m5ubAgMD7T4XMm7cOGVkZNjLsWPHirh6AAAAALg8o5csmuLh4SEPDw/TZQAAAAAo50rtDFloaKgkKTU11ak9NTXV3hYaGqq0tDSn7efOndPJkyftPgAAAABQWpXaQFa7dm2FhoYqNjbWbsvMzNTXX3+tiIgISVJERITS09O1a9cuu8+nn36q/Px8tWrVqsRrBgAAAICrYfSSxaysLH3//ff2emJiouLj4xUYGKiwsDCNGDFCzz//vOrVq6fatWvrueeeU7Vq1dSrVy9JUqNGjdS5c2c98sgjWrhwoXJzczVs2DA98MADF33CIgAAAACUFkYD2c6dO3XHHXfY6yNHjpQkRUVFKSYmRv/85z91+vRpPfroo0pPT1fbtm21YcMGeXp62vssWbJEw4YNU8eOHeXi4qI+ffro1VdfLfFzAQAAAICr5bAsyzJdhGmZmZny9/dXRkaG/Pz8TJdTwLhVCUV2rGm9w4vsWLiGfDTcdAUAikOPOaYrAIAyp6SzQbl8yiIAAGVCefyfLYRQAGVMqX2oBwAAAACUdQQyAAAAADCEQAYAAAAAhhDIAAAAAMAQAhkAAAAAGEIgAwAAAABDCGQAAAAAYAiBDAAAAAAMIZABAAAAgCEEMgAAAAAwhEAGAAAAAIYQyAAAAADAEAIZAAAAABhCIAMAAAAAQ9xMF4CSNW5VgtP6tN7hhioBAAAAwAwZAAAAABhCIAMAAAAAQwhkAAAAAGAIgQwAAAAADCGQAQAAAIAhPGUR5ctHw01XAAAAANiYIQMAAAAAQwhkAAAAAGAIgQwAAAAADCGQAQAAAIAhBDIAAAAAMIRABgAAAACGEMgAAAAAwBACGQAAAAAYQiADAAAAAEMIZAAAAABgCIEMAAAAAAwhkAEAAACAIQQyAAAAADCEQAYAAAAAhhDIAAAAAMAQAhkAAAAAGOJmugAUNG5VgukSAAAAAJQAZsgAAAAAwBACGQAAAAAYQiADAAAAAEMIZAAAAABgCIEMAAAAAAzhKYsAAODa8dFw0xWY0WOO6QoAFBNmyAAAAADAEAIZAAAAABhCIAMAAAAAQwhkAAAAAGAIgQwAAAAADCGQAQAAAIAhBDIAAAAAMIRABgAAAACGEMgAAAAAwBACGQAAAAAYQiADAAAAAEMIZAAAAABgCIEMAAAAAAwhkAEAAACAIW6mC4BZ41YlOK1P6x1uqBIAAACg/GGGDAAAAAAMIZABAAAAgCEEMgAAAAAwhEAGAAAAAIYQyAAAAADAEAIZAAAAABhCIAMAAAAAQwhkAAAAAGAIgQwAAAAADCGQAQAAAIAhBDIAAAAAMIRABgAAAACGEMgAAAAAwBA30wWgdBm3KsFpfVrvcEOVAAAA20fDTVdgRo85pisAil2ZmSGbN2+eatWqJU9PT7Vq1Urbt283XVKZMG5VgtMCAAAAoOiUiUC2YsUKjRw5UhMmTNA333yjG2+8UZGRkUpLSzNdGgAAAABcVJm4ZHHWrFl65JFHNHjwYEnSwoULtXbtWr355pt6+umnDVcHAAAAXIXyeIlqOb489ZoPZDk5Odq1a5fGjRtnt7m4uKhTp06Ki4u74D7Z2dnKzs621zMyMiRJmZmZxVvsFco+k2W6hIsqLd9RoZ3JvnwfAABQOqx4zHQFKCml6HfM87/vWpZVIp93zQeyX375RXl5eQoJCXFqDwkJ0bfffnvBfaZNm6ZJkyYVaK9Ro0ax1FiWzDZdAAAAAMqgRaYLKODXX3+Vv79/sX/ONR/ICmPcuHEaOXKkvZ6fn6+TJ08qKChIDofDYGWQ/vi/EjVq1NCxY8fk5+dnuhyUIowNXAxjAxfD2MDFMDZwMRkZGQoLC1NgYGCJfN41H8gqV64sV1dXpaamOrWnpqYqNDT0gvt4eHjIw8PDqS0gIKC4SkQh+fn58R9IXBBjAxfD2MDFMDZwMYwNXIyLS8k8//Caf8qiu7u7WrRoodjYWLstPz9fsbGxioiIMFgZAAAAAFzaNT9DJkkjR45UVFSUWrZsqVtuuUWvvPKKTp8+bT91EQAAAABKozIRyO6//36dOHFC48ePV0pKim666SZt2LChwIM+cG3w8PDQhAkTClxWCjA2cDGMDVwMYwMXw9jAxZT02HBYJfU8RwAAAACAk2v+HjIAAAAAuFYRyAAAAADAEAIZAAAAABhCIAMAAAAAQwhkKBHTpk3TzTffrIoVKyo4OFi9evXSoUOHnPqcPXtW0dHRCgoKkq+vr/r06VPghd9JSUnq1q2bvL29FRwcrDFjxujcuXMleSooRtOnT5fD4dCIESPsNsZF+fXzzz/roYceUlBQkLy8vBQeHq6dO3fa2y3L0vjx41W1alV5eXmpU6dOOnz4sNMxTp48qf79+8vPz08BAQEaMmSIsrKySvpUUITy8vL03HPPqXbt2vLy8tL111+vKVOm6M/PKGNslB+fffaZevTooWrVqsnhcGj16tVO24tqLOzdu1ft2rWTp6enatSooRkzZhT3qeFvutTYyM3N1dixYxUeHi4fHx9Vq1ZNAwcO1PHjx52OUWJjwwJKQGRkpLV48WJr3759Vnx8vNW1a1crLCzMysrKsvs89thjVo0aNazY2Fhr586dVuvWra1bb73V3n7u3DmrSZMmVqdOnazdu3db69atsypXrmyNGzfOxCmhiG3fvt2qVauW1bRpU2v48OF2O+OifDp58qRVs2ZNa9CgQdbXX39tHTlyxPr444+t77//3u4zffp0y9/f31q9erW1Z88e6+6777Zq165t/f7773afzp07WzfeeKP11VdfWZ9//rlVt25dq1+/fiZOCUXkhRdesIKCgqw1a9ZYiYmJ1sqVKy1fX19rzpw5dh/GRvmxbt0665lnnrFWrVplSbLef/99p+1FMRYyMjKskJAQq3///ta+ffusZcuWWV5eXtaiRYtK6jRRCJcaG+np6VanTp2sFStWWN9++60VFxdn3XLLLVaLFi2cjlFSY4NABiPS0tIsSdbWrVsty/rjH0aFChWslStX2n0OHjxoSbLi4uIsy/rjH5aLi4uVkpJi91mwYIHl5+dnZWdnl+wJoEidOnXKqlevnrVx40br9ttvtwMZ46L8Gjt2rNW2bduLbs/Pz7dCQ0Otl156yW5LT0+3PDw8rGXLllmWZVkHDhywJFk7duyw+6xfv95yOBzWzz//XHzFo1h169bNevjhh53aevfubfXv39+yLMZGefbXX7qLaizMnz/fqlSpktPPlLFjx1oNGjQo5jNCUblQWP+r7du3W5Kso0ePWpZVsmODSxZhREZGhiQpMDBQkrRr1y7l5uaqU6dOdp+GDRsqLCxMcXFxkqS4uDiFh4c7vfA7MjJSmZmZ2r9/fwlWj6IWHR2tbt26Of39S4yL8uzDDz9Uy5Ytdd999yk4OFjNmjXT66+/bm9PTExUSkqK09jw9/dXq1atnMZGQECAWrZsaffp1KmTXFxc9PXXX5fcyaBI3XrrrYqNjdV3330nSdqzZ4+++OILdenSRRJjA/9VVGMhLi5Ot912m9zd3e0+kZGROnTokH777bcSOhsUt4yMDDkcDgUEBEgq2bHhVjSnAFy5/Px8jRgxQm3atFGTJk0kSSkpKXJ3d7f/EZwXEhKilJQUu8+ff+k+v/38Nlybli9frm+++UY7duwosI1xUX4dOXJECxYs0MiRI/W///u/2rFjh5588km5u7srKirK/ru90N/9n8dGcHCw03Y3NzcFBgYyNq5hTz/9tDIzM9WwYUO5uroqLy9PL7zwgvr37y9JjA3YimospKSkqHbt2gWOcX5bpUqViqV+lJyzZ89q7Nix6tevn/z8/CSV7NggkKHERUdHa9++ffriiy9MlwLDjh07puHDh2vjxo3y9PQ0XQ5Kkfz8fLVs2VJTp06VJDVr1kz79u3TwoULFRUVZbg6mPTee+9pyZIlWrp0qW644QbFx8drxIgRqlatGmMDwFXLzc1V3759ZVmWFixYYKQGLllEiRo2bJjWrFmjzZs3q3r16nZ7aGiocnJylJ6e7tQ/NTVVoaGhdp+/Pl3v/Pr5Pri27Nq1S2lpaWrevLnc3Nzk5uamrVu36tVXX5Wbm5tCQkIYF+VU1apV1bhxY6e2Ro0aKSkpSdJ//24v9Hf/57GRlpbmtP3cuXM6efIkY+MaNmbMGD399NN64IEHFB4ergEDBuipp57StGnTJDE28F9FNRb4OVN2nQ9jR48e1caNG+3ZMalkxwaBDCXCsiwNGzZM77//vj799NMC07stWrRQhQoVFBsba7cdOnRISUlJioiIkCRFREQoISHB6R/H+X88f/3FDdeGjh07KiEhQfHx8fbSsmVL9e/f3/4z46J8atOmTYFXY3z33XeqWbOmJKl27doKDQ11GhuZmZn6+uuvncZGenq6du3aZff59NNPlZ+fr1atWpXAWaA4nDlzRi4uzr++uLq6Kj8/XxJjA/9VVGMhIiJCn332mXJzc+0+GzduVIMGDbhc8Rp2PowdPnxYmzZtUlBQkNP2Eh0bV/UIEKCQhg4davn7+1tbtmyxkpOT7eXMmTN2n8cee8wKCwuzPv30U2vnzp1WRESEFRERYW8//3jzu+66y4qPj7c2bNhgValShceblzF/fsqiZTEuyqvt27dbbm5u1gsvvGAdPnzYWrJkieXt7W29++67dp/p06dbAQEB1gcffGDt3bvX6tmz5wUfZ92sWTPr66+/tr744gurXr16PNr8GhcVFWVdd9119mPvV61aZVWuXNn65z//afdhbJQfp06dsnbv3m3t3r3bkmTNmjXL2r17t/2kvKIYC+np6VZISIg1YMAAa9++fdby5cstb29vHntfyl1qbOTk5Fh33323Vb16dSs+Pt7pd9M/PzGxpMYGgQwlQtIFl8WLF9t9fv/9d+vxxx+3KlWqZHl7e1v33HOPlZyc7HScH3/80erSpYvl5eVlVa5c2Ro1apSVm5tbwmeD4vTXQMa4KL8++ugjq0mTJpaHh4fVsGFD61//+pfT9vz8fOu5556zQkJCLA8PD6tjx47WoUOHnPr8+uuvVr9+/SxfX1/Lz8/PGjx4sHXq1KmSPA0UsczMTGv48OFWWFiY5enpadWpU8d65plnnH6JYmyUH5s3b77g7xdRUVGWZRXdWNizZ4/Vtm1by8PDw7ruuuus6dOnl9QpopAuNTYSExMv+rvp5s2b7WOU1NhwWNafXm0PAAAAACgx3EMGAAAAAIYQyAAAAADAEAIZAAAAABhCIAMAAAAAQwhkAAAAAGAIgQwAAAAADCGQAQAAAIAhBDIAAAAAMIRABgAo1X788Uc5HA7Fx8ebLgUAgCJHIAMAFDuHw3HJZeLEiaZLvKDvv/9egwcPVvXq1eXh4aHatWurX79+2rlzZ4nWQSgFgLLLzXQBAICyLzk52f7zihUrNH78eB06dMhu8/X1NVHWJe3cuVMdO3ZUkyZNtGjRIjVs2FCnTp3SBx98oFGjRmnr1q2mSwQAlAHMkAEAil1oaKi9+Pv7y+Fw2OvBwcGaNWuWPQt10003acOGDRc9Vl5enh5++GE1bNhQSUlJkqQPPvhAzZs3l6enp+rUqaNJkybp3Llz9j4Oh0P//ve/dc8998jb21v16tXThx9+eNHPsCxLgwYNUr169fT555+rW7duuv7663XTTTdpwoQJ+uCDD+y+CQkJ6tChg7y8vBQUFKRHH31UWVlZ9vb27dtrxIgRTsfv1auXBg0aZK/XqlVLU6dO1cMPP6yKFSsqLCxM//rXv+zttWvXliQ1a9ZMDodD7du3v+T3DQC4dhDIAABGzZkzRzNnztTLL7+svXv3KjIyUnfffbcOHz5coG92drbuu+8+xcfH6/PPP1dYWJg+//xzDRw4UMOHD9eBAwe0aNEixcTE6IUXXnDad9KkSerbt6/27t2rrl27qn///jp58uQFa4qPj9f+/fs1atQoubgU/FEZEBAgSTp9+rQiIyNVqVIl7dixQytXrtSmTZs0bNiwq/4eZs6cqZYtW2r37t16/PHHNXToUHsWcfv27ZKkTZs2KTk5WatWrbrq4wMASicCGQDAqJdfflljx47VAw88oAYNGujFF1/UTTfdpFdeecWpX1ZWlrp166YTJ05o8+bNqlKliqQ/gtbTTz+tqKgo1alTR3feeaemTJmiRYsWOe0/aNAg9evXT3Xr1tXUqVOVlZVlB52/Oh8GGzZseMnaly5dqrNnz+rtt99WkyZN1KFDB82dO1fvvPOOUlNTr+p76Nq1qx5//HHVrVtXY8eOVeXKlbV582ZJss81KChIoaGhCgwMvKpjAwBKL+4hAwAYk5mZqePHj6tNmzZO7W3atNGePXuc2vr166fq1avr008/lZeXl92+Z88effnll04zYnl5eTp79qzOnDkjb29vSVLTpk3t7T4+PvLz81NaWtoF67Is64rqP3jwoG688Ub5+Pg41Z6fn69Dhw4pJCTkio7z1/rOX9J5sfoAAGUHM2QAgGtC165dtXfvXsXFxTm1Z2VladKkSYqPj7eXhIQEHT58WJ6enna/ChUqOO3ncDiUn59/wc+qX7++JOnbb7/923W7uLgUCHi5ubkF+l1NfQCAsoNABgAwxs/PT9WqVdOXX37p1P7ll1+qcePGTm1Dhw7V9OnTdffddzs94bB58+Y6dOiQ6tatW2C50P1fV+Kmm25S48aNNXPmzAuGovT0dElSo0aNtGfPHp0+fdqpdhcXFzVo0EDSH5cb/vkpk3l5edq3b99V1ePu7m7vCwAoWwhkAACjxowZoxdffFErVqzQoUOH9PTTTys+Pl7Dhw8v0PeJJ57Q888/r+7du+uLL76QJI0fP15vv/22Jk2apP379+vgwYNavny5nn322ULX5HA4tHjxYn333Xdq166d1q1bpyNHjmjv3r164YUX1LNnT0lS//795enpqaioKO3bt0+bN2/WE088oQEDBtiXK3bo0EFr167V2rVr9e2332ro0KF2oLtSwcHB8vLy0oYNG5SamqqMjIxCnxsAoHQhkAEAjHryySc1cuRIjRo1SuHh4dqwYYM+/PBD1atX74L9R4wYoUmTJqlr167atm2bIiMjtWbNGn3yySe6+eab1bp1a82ePVs1a9b8W3Xdcsst2rlzp+rWratHHnlEjRo10t133639+/fbDxzx9vbWxx9/rJMnT+rmm2/Wvffeq44dO2ru3Ln2cR5++GFFRUVp4MCBuv3221WnTh3dcccdV1WLm5ubXn31VS1atEjVqlWzAyEA4NrnsK70zmUAAAAAQJFihgwAAAAADCGQAQAAAIAhBDIAAAAAMIRABgAAAACGEMgAAAAAwBACGQAAAAAYQiADAAAAAEMIZAAAAABgCIEMAAAAAAwhkAEAAACAIQQyAAAAADDk/wHjukUbR19+3QAAAABJRU5ErkJggg==",
+      "text/plain": [
+       "<Figure size 1000x600 with 1 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "# Plot the histograms\n",
+    "plt.figure(figsize=(10, 6))\n",
+    "\n",
+    "# Histogram for Input Tokens\n",
+    "plt.hist(df_token_mistral['input_tokens'], bins=10, alpha=0.6, label='Input Tokens')\n",
+    "\n",
+    "# Histogram for Output Tokens\n",
+    "plt.hist(df_token_mistral['output_tokens'], bins=10, alpha=0.6, label='Output Tokens')\n",
+    "\n",
+    "# Add titles and labels\n",
+    "plt.title(\"Token Summary\")\n",
+    "plt.xlabel(\"Token Count\")\n",
+    "plt.ylabel(\"Frequency\")\n",
+    "plt.legend()\n",
+    "\n",
+    "# Show the plot\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "d6a78d92-2fc4-4354-8825-b17cba59eee4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Our Max Input Tokens:\t162\n",
+      "Our Max Output Tokens:\t1148\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(f\"Our Max Input Tokens:\\t{max(df_token_mistral.input_tokens)}\\nOur Max Output Tokens:\\t{max(df_token_mistral.output_tokens)}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fdfc0581-1c57-436c-8c76-9bfeab278603",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

03-poe-eval-sg.ipynb

@@ -0,0 +1,1981 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "7eb11cd7-6200-45f9-822f-45b8b392c4bd",
+   "metadata": {},
+   "source": [
+    "# Setup"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4ca90151-0b89-4870-8213-9d49cb68c555",
+   "metadata": {},
+   "source": [
+    "## Config\n",
+    "Set the tokens based on the numbers in [02-poe-token-count-exploration.ipynb](02-poe-token-count-exploration.ipynb). I like to give a little buffer in-case an explanation goes over."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "5238c6e9-9425-4ced-a16a-998e775e7342",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "INPUT_TOKENS = 300\n",
+    "OUTPUT_TOKENS = 1600\n",
+    "\n",
+    "INPUT_DATASET = 'derek-thomas/labeled-multiple-choice-explained-mistral-tokenized'\n",
+    "OUTPUT_DATASET = 'derek-thomas/labeled-multiple-choice-explained-mistral-results'\n",
+    "BASE_MODEL = 'mistralai/Mistral-7B-Instruct-v0.3'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f4eca659-f11f-4d25-886a-9d7af4f38411",
+   "metadata": {},
+   "source": [
+    "# Setup\n",
+    "Here we create the pydantic models for each of our experiments. Note because of how you specify field names in pydantic, we need to use an `alias` and `populate_by_name`. Given that our `Final Answer` is always a letter between a-h we can use an enumeration."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "c5367700-0e9d-435b-875a-02a73b292ade",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pydantic import BaseModel, Field\n",
+    "from typing import List\n",
+    "from enum import Enum\n",
+    "import json\n",
+    "\n",
+    "\n",
+    "class FinalAnswerEnum(str, Enum):\n",
+    "    a = \"a\"\n",
+    "    b = \"b\"\n",
+    "    c = \"c\"\n",
+    "    d = \"d\"\n",
+    "    e = \"e\"\n",
+    "    f = \"f\"\n",
+    "    g = \"g\"\n",
+    "    h = \"h\"\n",
+    "\n",
+    "class RFAModel(BaseModel):\n",
+    "    reasoning: str = Field(..., alias=\"Reasoning\")\n",
+    "    final_answer: FinalAnswerEnum = Field(..., alias=\"Final Answer\")\n",
+    "\n",
+    "    class Config:\n",
+    "        populate_by_name = True\n",
+    "        \n",
+    "class FARModel(BaseModel):\n",
+    "    final_answer: FinalAnswerEnum = Field(..., alias=\"Final Answer\")\n",
+    "    reasoning: str = Field(..., alias=\"Reasoning\")\n",
+    "\n",
+    "    class Config:\n",
+    "        populate_by_name = True\n",
+    "        \n",
+    "class FAModel(BaseModel):\n",
+    "    final_answer: FinalAnswerEnum = Field(..., alias=\"Final Answer\")\n",
+    "\n",
+    "    class Config:\n",
+    "        populate_by_name = True"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7e0f51c0-c4f7-4299-9a24-a4a90d4a9f2a",
+   "metadata": {},
+   "source": [
+    "We generated lots of experiments in [derek-thomas/labeled-multiple-choice-explained-mistral-tokenized](https://huggingface.co/datasets/derek-thomas/labeled-multiple-choice-explained-mistral-tokenized/viewer?row=0). Now we will aggregate everything we need in `experiments` for convenience."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "5d0bd22f-293e-4c15-9dfe-8070553f42b5",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'derek-thomas/mistral-v03-poe-RFA-mistral,derek-thomas/mistral-v03-poe-FAR-mistral,derek-thomas/mistral-v03-poe-RFA-gpt3-5,derek-thomas/mistral-v03-poe-FAR-gpt3-5,derek-thomas/mistral-v03-poe-FAR'"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "\n",
+    "experiments = {\n",
+    "    'RFA-mistral': {\n",
+    "        'pydantic': RFAModel,\n",
+    "        \"lora\": \"derek-thomas/mistral-v03-poe-RFA-mistral\",\n",
+    "        \"column\": 'user_prompt_RFA',\n",
+    "    },\n",
+    "    'FAR-mistral': {\n",
+    "        'pydantic': FARModel,\n",
+    "        \"lora\": \"derek-thomas/mistral-v03-poe-FAR-mistral\",\n",
+    "        \"column\": 'user_prompt_FAR',\n",
+    "    },\n",
+    "    'RFA-gpt3-5': {\n",
+    "        'pydantic': RFAModel,\n",
+    "        \"lora\": \"derek-thomas/mistral-v03-poe-RFA-gpt3-5\",\n",
+    "        \"column\": 'user_prompt_RFA',\n",
+    "    },\n",
+    "    'FAR-gpt3-5': {\n",
+    "        'pydantic': FARModel,\n",
+    "        \"lora\": \"derek-thomas/mistral-v03-poe-FAR-gpt3-5\",\n",
+    "        \"column\": 'user_prompt_FAR',\n",
+    "    },\n",
+    "    'FA': {\n",
+    "        'pydantic': FAModel,\n",
+    "        \"lora\": \"derek-thomas/mistral-v03-poe-FAR\",\n",
+    "        \"column\": 'user_prompt_FA',\n",
+    "    },\n",
+    "    'base': {\n",
+    "        'pydantic': FAModel,\n",
+    "        \"lora\": None,\n",
+    "        \"column\": 'user_prompt_FA',\n",
+    "    },\n",
+    "}\n",
+    "\n",
+    "LORAS_STRING = ','.join([v['lora'] for _, v in experiments.items() if v and v.get('lora') is not None])\n",
+    "LORAS_STRING"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "6f8826fb-76ea-464f-8146-262bda0b58bc",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f9a6617489af4d86be59e614e3c505c4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from huggingface_hub import login, get_token\n",
+    "\n",
+    "# Log in to your Hugging Face account\n",
+    "login()  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "a9e2d29c-1f8e-4a70-839f-f61ae396d6f6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoTokenizer\n",
+    "\n",
+    "tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, token=get_token())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "3aa16403-88e4-4eba-a45b-3d91db9bb58d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from datasets import load_dataset\n",
+    "\n",
+    "# Load dataset (test split)\n",
+    "dataset = load_dataset(INPUT_DATASET, split='test')\n",
+    "df = dataset.to_pandas()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c787fbb8-beae-4bb0-860d-0ce4d0c9b9a4",
+   "metadata": {},
+   "source": [
+    "# Evaluation"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a35d4955-61e5-4143-ab92-7e9b8a49ea1b",
+   "metadata": {},
+   "source": [
+    "## Endpoint Configuration\n",
+    "Im using 8 replicas so I can move quickly! The try/except is in-case I need to make manual changes and I want to load the endpoint."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "619fbe5e-e35d-430b-a617-9223ba6babd0",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "from huggingface_hub import create_inference_endpoint\n",
+    "from huggingface_hub import get_inference_endpoint\n",
+    "\n",
+    "\n",
+    "def get_my_endpoint():\n",
+    "    name = f\"prompt-order-experiment\"\n",
+    "    namespace='HF-test-lab'\n",
+    "    try:\n",
+    "        endpoint = get_inference_endpoint(name, namespace=namespace)\n",
+    "        endpoint.wait()\n",
+    "    except:\n",
+    "        # Custom Docker image details\n",
+    "        custom_image = {\n",
+    "            \"health_route\": \"/health\",\n",
+    "            \"url\": \"ghcr.io/huggingface/text-generation-inference:sha-caff779\",  # Needs to be >=2.4.2 to get ordering of json outputs\n",
+    "            \"env\": {\n",
+    "                \"LORA_ADAPTERS\": LORAS_STRING,\n",
+    "                \"MAX_BATCH_PREFILL_TOKENS\": str(20*INPUT_TOKENS),\n",
+    "                \"MAX_INPUT_TOKENS\": str(INPUT_TOKENS), \n",
+    "                \"MAX_TOTAL_TOKENS\": str(INPUT_TOKENS + OUTPUT_TOKENS), \n",
+    "                \"DISABLE_CUSTOM_KERNELS\": 'false',\n",
+    "                \"MODEL_ID\": \"/repository\"\n",
+    "            },\n",
+    "        }\n",
+    "        \n",
+    "        secrets = {\n",
+    "            \"HF_TOKEN\": get_token()\n",
+    "        }\n",
+    "        \n",
+    "        # Creating the inference endpoint\n",
+    "        endpoint = create_inference_endpoint(\n",
+    "            name=name,\n",
+    "            namespace=namespace,\n",
+    "            repository='mistralai/Mistral-7B-Instruct-v0.3',\n",
+    "            framework=\"pytorch\",\n",
+    "            accelerator=\"gpu\",\n",
+    "            instance_size=\"x1\",\n",
+    "            instance_type=\"nvidia-l4\",\n",
+    "            region=\"us-east-1\",\n",
+    "            vendor=\"aws\",\n",
+    "            min_replica=8,\n",
+    "            max_replica=8,\n",
+    "            task=\"text-generation\",\n",
+    "            custom_image=custom_image,\n",
+    "            secrets=secrets\n",
+    "        )\n",
+    "        # endpoint.wait()\n",
+    "            \n",
+    "    print(\"Your model is ready to use!\")\n",
+    "    endpoint.wait()\n",
+    "    return endpoint"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "000b907a-224d-4dbf-aa0d-e0dbee1b8787",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Your model is ready to use!\n",
+      "CPU times: user 22.3 ms, sys: 7.64 ms, total: 30 ms\n",
+      "Wall time: 2.07 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "endpoint = get_my_endpoint()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5708f348-c11e-4b66-aeff-93f5ec08ab49",
+   "metadata": {},
+   "source": [
+    "## Manual Evaluation\n",
+    "Since we havent seen our models in use yet, its a good time to check them out!"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "328dd842-eaa4-470c-b761-7c403b453321",
+   "metadata": {},
+   "source": [
+    "### Reasoning Final Answer\n",
+    "In both mistral and gpt3-5 we should see the **Reasoning** first and then the **Final Answer** in the prompt and the responses."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "47af6191-7765-4047-bd8f-64aadb08434e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'<s>[INST] Answer the Question and include your Reasoning and the Final Answer in a json like: {\"Reasoning: \"...\", \"Final Answer\": \"x\"} where x is a letter that corresponds to the answer choice which is a letter between a and h.\\nQuestion: What are busses used for?\\nAnswer Choices: (a) Protective shelter (b) Transporting humans (c) Help other species benefit (d) Transporting airplanes (e) A backbone (f) Communication (g) Safe operation (h) Safe driving[/INST]'"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "key = 'RFA-mistral'\n",
+    "user_prompt_RFA = df.iloc[0][experiments[key]['column']]\n",
+    "user_prompt_RFA"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "1f976218-f33c-4db3-9797-3935e121e6b2",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'{\"Reasoning\": \"Busses are primarily used for transporting humans, so the correct answer is (b) Transporting humans. The other options are either incorrect (a, c, d, e, f, g, h) or not specific enough to the function of a bus (a, c, e, f, g, h).\", \"Final Answer\": \"b\"}'"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "response = endpoint.client.text_generation(\n",
+    "    prompt=user_prompt_RFA,\n",
+    "    max_new_tokens=OUTPUT_TOKENS,\n",
+    "    adapter_id=experiments[key]['lora'],\n",
+    "    grammar={\"type\": \"json\", \"value\": experiments[key]['pydantic'].schema()},\n",
+    ")\n",
+    "response"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "222e33b7-0158-44f8-8848-da5318e699b4",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'{\"Reasoning\": \"Busses are primarily used for transporting humans, especially in urban areas, schools, and tourist destinations. They provide a means of public transportation, making it easier for people to travel to various locations without the need for personal vehicles. Therefore, the correct answer is (b) transporting humans.\", \"Final Answer\": \"b\"}'"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "key = 'RFA-gpt3-5'\n",
+    "response = endpoint.client.text_generation(\n",
+    "    prompt=user_prompt_RFA,\n",
+    "    max_new_tokens=575,\n",
+    "    adapter_id=experiments[key]['lora'],\n",
+    "    grammar={\"type\": \"json\", \"value\": experiments[key]['pydantic'].schema()},\n",
+    ")\n",
+    "response"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "138fd46d-896f-4b4b-90d3-d3fd7075f149",
+   "metadata": {},
+   "source": [
+    "### Final Answer Reasoning \n",
+    "In both mistral and gpt3-5 we should see the **Final Answer** first and then the **Reasoning** in the prompt and the responses."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "ec3e574b-f63f-4513-a6ae-335136543a8c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'<s>[INST] Answer the Question and include your Final Answer and the Reasoning in a json like: {\"Final Answer\": \"x\", \"Reasoning: \"...\"} where x is a letter that corresponds to the answer choice which is a letter between a and h.\\nQuestion: What are busses used for?\\nAnswer Choices: (a) Protective shelter (b) Transporting humans (c) Help other species benefit (d) Transporting airplanes (e) A backbone (f) Communication (g) Safe operation (h) Safe driving[/INST]'"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "key = 'FAR-gpt3-5'\n",
+    "user_prompt_FAR = df.iloc[0][experiments[key]['column']]\n",
+    "user_prompt_FAR"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "24f30a15-5ec0-4f26-b32f-b4ccb429e6f9",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'{\"Final Answer\": \"b\", \"Reasoning\": \"Busses are primarily used for transporting humans, especially in urban areas, to facilitate public transportation. They provide a means of transportation for a large number of people at once, reducing the number of vehicles on the road and helping to alleviate traffic congestion. They also serve as a protective shelter for passengers, shielding them from the elements during travel.\"}'"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "response = endpoint.client.text_generation(\n",
+    "    prompt=user_prompt_FAR,\n",
+    "    max_new_tokens=575,\n",
+    "    adapter_id=experiments[key]['lora'],\n",
+    "    grammar={\"type\": \"json\", \"value\": experiments[key]['pydantic'].schema()},\n",
+    ")\n",
+    "response"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "32536844-211d-4856-983c-d5787734d420",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'{\"Final Answer\": \"b\", \"Reasoning\": \"Busses are primarily used for transporting humans. While they can provide a protective shelter and ensure safe operation, they are not used for transporting airplanes, helping other species benefit, serving as a backbone, or being used for communication.\"}'"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "key = 'FAR-mistral'\n",
+    "response = endpoint.client.text_generation(\n",
+    "    prompt=user_prompt_FAR,\n",
+    "    max_new_tokens=575,\n",
+    "    adapter_id=experiments[key]['lora'],\n",
+    "    grammar={\"type\": \"json\", \"value\": experiments[key]['pydantic'].schema()},\n",
+    ")\n",
+    "response"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5482cf12-880c-4112-ba87-059a03a3f466",
+   "metadata": {},
+   "source": [
+    "### Final Answer \n",
+    "Here we should juse see the **Final Answer** and no **Reasoning**."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "71a9f634-319c-40c2-8f66-18e282732320",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'<s>[INST] Answer the Question and include your Final Answer in a json like: {\"Final Answer\": \"x\"} where x is a letter that corresponds to the answer choice which is a letter between a and h.\\nQuestion: What are busses used for?\\nAnswer Choices: (a) Protective shelter (b) Transporting humans (c) Help other species benefit (d) Transporting airplanes (e) A backbone (f) Communication (g) Safe operation (h) Safe driving[/INST]'"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "key = 'FA'\n",
+    "user_prompt_FA = df.iloc[0][experiments[key]['column']]\n",
+    "user_prompt_FA"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "1cded37d-b907-4f4d-9b8c-c2167c6ba213",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'{\"Final Answer\": \"b\"}'"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "response = endpoint.client.text_generation(\n",
+    "    prompt=user_prompt_FA,\n",
+    "    max_new_tokens=575,\n",
+    "    adapter_id=experiments[key]['lora'],\n",
+    ")\n",
+    "response"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "122b8563-3220-43ac-9e0d-ef84ebcbb9e1",
+   "metadata": {},
+   "source": [
+    "## Evaluation Loop\n",
+    "I used 20x the prefill than the input and 8 replicas so I should capacity for ~160 parallel requests. Im only using 128 but it should be pretty fast."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "fbc69ea7-e2ca-4405-82f0-5af95375ec88",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import nest_asyncio\n",
+    "import asyncio\n",
+    "from transformers import AutoTokenizer\n",
+    "from tqdm.auto import tqdm\n",
+    "\n",
+    "# Allow nested event loops in Jupyter\n",
+    "nest_asyncio.apply()\n",
+    "\n",
+    "\n",
+    "# Semaphore to limit concurrency\n",
+    "CONCURRENCY_LIMIT = 100 \n",
+    "MAX_NEW_TOKENS = OUTPUT_TOKENS\n",
+    "semaphore = asyncio.Semaphore(CONCURRENCY_LIMIT)\n",
+    "\n",
+    "# Progress bar\n",
+    "progress_bar = None  # Global to allow updates from within async functions\n",
+    "\n",
+    "# Function to send asynchronous requests to the endpoint\n",
+    "async def fetch_response_async(async_client, prompt, lora_id, pydantic_model):\n",
+    "    async with semaphore:  # Limit the number of concurrent requests\n",
+    "        response = await async_client.text_generation(\n",
+    "            prompt=prompt,\n",
+    "            max_new_tokens=MAX_NEW_TOKENS,\n",
+    "            adapter_id=lora_id if lora_id else None,\n",
+    "            grammar={\"type\": \"json\", \"value\": pydantic_model.schema()}\n",
+    "        )\n",
+    "        progress_bar.update(1)  # Update the progress bar when the request is complete\n",
+    "        return response\n",
+    "\n",
+    "# Function to process a single conversation type asynchronously\n",
+    "async def process_conversation_type(conversation_type, model_info, df, tokenizer, async_client):\n",
+    "    response_column = f\"responses_{conversation_type.replace('-','_')}\"\n",
+    "    responses = []  # Temporary list to hold responses for the current conversation type\n",
+    "\n",
+    "    tasks = []\n",
+    "    for _, item in df.iterrows():\n",
+    "        prompt = item.get(model_info[\"column\"])\n",
+    "        tasks.append(fetch_response_async(async_client, prompt, model_info[\"lora\"], model_info[\"pydantic\"]))\n",
+    "\n",
+    "    # Wait for all tasks to complete\n",
+    "    responses = await asyncio.gather(*tasks)\n",
+    "\n",
+    "    # If responses are strings, use them directly; otherwise, extract 'generated_text'\n",
+    "    try:\n",
+    "        df[response_column] = [resp[\"generated_text\"] for resp in responses]\n",
+    "    except TypeError:  # Fallback in case responses are raw strings\n",
+    "        df[response_column] = responses\n",
+    "\n",
+    "# Main function to handle all conversation types\n",
+    "async def main(df, models, tokenizer, async_client):\n",
+    "    global progress_bar\n",
+    "    total_requests = len(df) * len(models)  # Total number of requests across all conversation types\n",
+    "    progress_bar = tqdm(total=total_requests, desc=\"Processing requests\")\n",
+    "\n",
+    "    tasks = []\n",
+    "    for conversation_type, model_info in models.items():\n",
+    "        tasks.append(process_conversation_type(conversation_type, model_info, df, tokenizer, async_client))\n",
+    "    await asyncio.gather(*tasks)\n",
+    "\n",
+    "    progress_bar.close()  # Close the progress bar when done\n",
+    "\n",
+    "# Define parameters and run\n",
+    "# await main(df, experiments, tokenizer, endpoint.async_client)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9662adde-4ea4-4254-a6be-c619ed9557c8",
+   "metadata": {},
+   "source": [
+    "This is the same as above but with a couple nice features like time-out in case you run into any issues."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "e2671c78-6cd1-4bb7-aa04-b695bfb02115",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f29c84d7a4654e9aa6af47cfe1bbde39",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Processing requests:   0%|          | 0/10098 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import nest_asyncio\n",
+    "import asyncio\n",
+    "from transformers import AutoTokenizer\n",
+    "from tqdm.auto import tqdm\n",
+    "import time\n",
+    "\n",
+    "# Allow nested event loops in Jupyter\n",
+    "nest_asyncio.apply()\n",
+    "\n",
+    "# Semaphore to limit concurrency\n",
+    "CONCURRENCY_LIMIT = 100 \n",
+    "MAX_NEW_TOKENS = OUTPUT_TOKENS\n",
+    "semaphore = asyncio.Semaphore(CONCURRENCY_LIMIT)\n",
+    "\n",
+    "# Progress bar\n",
+    "progress_bar = None  # Global to allow updates from within async functions\n",
+    "\n",
+    "# Retry parameters\n",
+    "MAX_RETRIES = 3\n",
+    "BACKOFF_TIME = 2  # Time in seconds before retrying\n",
+    "\n",
+    "# Function to send asynchronous requests to the endpoint with retries\n",
+    "async def fetch_response_async(async_client, prompt, lora_id, pydantic_model):\n",
+    "    retries = 0\n",
+    "    while retries < MAX_RETRIES:\n",
+    "        try:\n",
+    "            async with semaphore:  # Limit the number of concurrent requests\n",
+    "                response = await async_client.text_generation(\n",
+    "                    prompt=prompt,\n",
+    "                    max_new_tokens=MAX_NEW_TOKENS,\n",
+    "                    adapter_id=lora_id if lora_id else None,\n",
+    "                    grammar={\"type\": \"json\", \"value\": pydantic_model.schema()}\n",
+    "                )\n",
+    "                progress_bar.update(1)  # Update the progress bar when the request is complete\n",
+    "                return response\n",
+    "        except Exception as e:\n",
+    "            retries += 1\n",
+    "            if retries >= MAX_RETRIES:\n",
+    "                raise e  # If we've exhausted retries, re-raise the error\n",
+    "            else:\n",
+    "                print(f\"Error: {e}. Retrying... ({retries}/{MAX_RETRIES})\")\n",
+    "                await asyncio.sleep(BACKOFF_TIME)  # Wait before retrying\n",
+    "\n",
+    "# Function to process a single conversation type asynchronously\n",
+    "async def process_conversation_type(conversation_type, model_info, df, tokenizer, async_client):\n",
+    "    response_column = f\"responses_{conversation_type.replace('-','_')}\"\n",
+    "    responses = []  # Temporary list to hold responses for the current conversation type\n",
+    "\n",
+    "    tasks = []\n",
+    "    for _, item in df.iterrows():\n",
+    "        prompt = item.get(model_info[\"column\"])\n",
+    "        tasks.append(fetch_response_async(async_client, prompt, model_info[\"lora\"], model_info[\"pydantic\"]))\n",
+    "\n",
+    "    # Wait for all tasks to complete\n",
+    "    responses = await asyncio.gather(*tasks)\n",
+    "\n",
+    "    # If responses are strings, use them directly; otherwise, extract 'generated_text'\n",
+    "    try:\n",
+    "        df[response_column] = [resp[\"generated_text\"] for resp in responses]\n",
+    "    except TypeError:  # Fallback in case responses are raw strings\n",
+    "        df[response_column] = responses\n",
+    "\n",
+    "# Main function to handle all conversation types\n",
+    "async def main(df, models, tokenizer, async_client):\n",
+    "    global progress_bar\n",
+    "    total_requests = len(df) * len(models)  # Total number of requests across all conversation types\n",
+    "    progress_bar = tqdm(total=total_requests, desc=\"Processing requests\")\n",
+    "\n",
+    "    tasks = []\n",
+    "    for conversation_type, model_info in models.items():\n",
+    "        tasks.append(process_conversation_type(conversation_type, model_info, df, tokenizer, async_client))\n",
+    "    await asyncio.gather(*tasks)\n",
+    "\n",
+    "    progress_bar.close()  # Close the progress bar when done\n",
+    "\n",
+    "# Define parameters and run\n",
+    "await main(df, experiments, tokenizer, endpoint.async_client)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d562e0b4-96ae-4259-925d-4d62b8c49641",
+   "metadata": {},
+   "source": [
+    "It took `00:10:43`. Not bad! That should be around `$1.14` total at `$0.80/gpu/hr`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "8f81466e-80fb-4915-9c68-dfbe168e052b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "InferenceEndpoint(name='prompt-order-experiment', namespace='HF-test-lab', repository='mistralai/Mistral-7B-Instruct-v0.3', status='paused', url=None)"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "endpoint.pause()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "f5a79dad-2475-4324-8a2b-77b33e9c0822",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_backup = df.copy()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "3473d555-927a-49ab-8e15-9097ed455c48",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>topic</th>\n",
+       "      <th>question_text</th>\n",
+       "      <th>answer_key</th>\n",
+       "      <th>gpt3_5_reasoning</th>\n",
+       "      <th>mistral_reasoning</th>\n",
+       "      <th>answer_choices</th>\n",
+       "      <th>user_prompt</th>\n",
+       "      <th>user_prompt_RFA</th>\n",
+       "      <th>conversation_RFA_gpt3_5</th>\n",
+       "      <th>conversation_RFA_mistral</th>\n",
+       "      <th>...</th>\n",
+       "      <th>conversation_FAR_gpt3_5</th>\n",
+       "      <th>conversation_FAR_mistral</th>\n",
+       "      <th>user_prompt_FA</th>\n",
+       "      <th>conversation_FA</th>\n",
+       "      <th>responses_RFA_mistral</th>\n",
+       "      <th>responses_FAR_mistral</th>\n",
+       "      <th>responses_RFA_gpt3_5</th>\n",
+       "      <th>responses_FAR_gpt3_5</th>\n",
+       "      <th>responses_FA</th>\n",
+       "      <th>responses_base</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Transportation</td>\n",
+       "      <td>What are busses used for?</td>\n",
+       "      <td>b</td>\n",
+       "      <td>a) Protective shelter: This option is incorrec...</td>\n",
+       "      <td>1. Start by reading the question carefully: \"C...</td>\n",
+       "      <td>(a) Protective shelter (b) Transporting humans...</td>\n",
+       "      <td>Question: What are busses used for?\\nAnswer Ch...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>{\"Reasoning\": \"Busses are primarily used for t...</td>\n",
+       "      <td>{\"Final Answer\": \"b\", \"Reasoning\": \"Busses are...</td>\n",
+       "      <td>{\"Reasoning\": \"Busses are primarily used for t...</td>\n",
+       "      <td>{\"Final Answer\": \"b\", \"Reasoning\": \"Busses are...</td>\n",
+       "      <td>{\"Final Answer\": \"b\"}</td>\n",
+       "      <td>{\"Final Answer\": \"b\"}</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Climate change</td>\n",
+       "      <td>Which of the following does not contribute to ...</td>\n",
+       "      <td>g</td>\n",
+       "      <td>a) Nucleus of a cell: This option is not relat...</td>\n",
+       "      <td>To solve this question, let's first understand...</td>\n",
+       "      <td>(a) Nucleus of a cell (b) Flying in a plane (c...</td>\n",
+       "      <td>Question: Which of the following does not cont...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>{\"Reasoning\": \"Global warming is primarily cau...</td>\n",
+       "      <td>{\"Final Answer\": \"a\", \"Reasoning\": \"The nucleu...</td>\n",
+       "      <td>{\"Reasoning\": \"The nucleus of a cell (option a...</td>\n",
+       "      <td>{\"Final Answer\": \"a\", \"Reasoning\": \"The nucleu...</td>\n",
+       "      <td>{\"Final Answer\": \"a\"}</td>\n",
+       "      <td>{\"Final Answer\": \"a\"}</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Photography</td>\n",
+       "      <td>What uses electrical energy converted from che...</td>\n",
+       "      <td>b</td>\n",
+       "      <td>a) Sunlight: Sunlight is a form of energy that...</td>\n",
+       "      <td>1. Read the question and options carefully: Th...</td>\n",
+       "      <td>(a) Sunlight (b) Cameras (c) Cells (d) Buses (...</td>\n",
+       "      <td>Question: What uses electrical energy converte...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>{\"Reasoning\": \"The question asks for an organi...</td>\n",
+       "      <td>{\"Final Answer\": \"c\", \"Reasoning\": \"The proces...</td>\n",
+       "      <td>{\"Reasoning\": \"The process of converting chemi...</td>\n",
+       "      <td>{\"Final Answer\": \"e\", \"Reasoning\": \"Bacteria u...</td>\n",
+       "      <td>{\"Final Answer\": \"c\"}</td>\n",
+       "      <td>{\"Final Answer\": \"c\"}</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Microbiology</td>\n",
+       "      <td>Bacteria causes what to be harmed?</td>\n",
+       "      <td>a</td>\n",
+       "      <td>Now, let's go through each option and explain ...</td>\n",
+       "      <td>To answer this question correctly, let's follo...</td>\n",
+       "      <td>(a) Plants (b) Electronics (c) Fossils (d) Hum...</td>\n",
+       "      <td>Question: Bacteria causes what to be harmed?\\n...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>{\"Reasoning\": \"Bacteria are microorganisms tha...</td>\n",
+       "      <td>{\"Final Answer\": \"d\", \"Reasoning\": \"Bacteria c...</td>\n",
+       "      <td>{ \"Reasoning\": \"Bacteria can cause harm to var...</td>\n",
+       "      <td>{\"Final Answer\": \"d\", \"Reasoning\": \"Bacteria c...</td>\n",
+       "      <td>{\"Final Answer\": \"d\"}</td>\n",
+       "      <td>{\"Final Answer\": \"d\"}</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Biology</td>\n",
+       "      <td>Plants and snakes live _.?</td>\n",
+       "      <td>a</td>\n",
+       "      <td>b) Important habitats: This option is incorrec...</td>\n",
+       "      <td>1. Read the question and options carefully: Th...</td>\n",
+       "      <td>(a) Almost everywhere (b) Important habitats (...</td>\n",
+       "      <td>Question: Plants and snakes live _.?\\nAnswer C...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>{\"Reasoning\": \"The question asks about the liv...</td>\n",
+       "      <td>{\"Final Answer\": \"a\", \"Reasoning\": \"Plants and...</td>\n",
+       "      <td>{\"Reasoning\": \"The question asks about the rel...</td>\n",
+       "      <td>{\"Final Answer\": \"f\", \"Reasoning\": \"Plants and...</td>\n",
+       "      <td>{\"Final Answer\": \"g\"}</td>\n",
+       "      <td>{\"Final Answer\": \"g\"}</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1678</th>\n",
+       "      <td>Biology</td>\n",
+       "      <td>New resources required for creation can be red...</td>\n",
+       "      <td>g</td>\n",
+       "      <td>a) Mining: Mining involves extracting minerals...</td>\n",
+       "      <td>1. Start by reading the question and options c...</td>\n",
+       "      <td>(a) Mining (b) Mutations (c) Fossil fuels (d) ...</td>\n",
+       "      <td>Question: New resources required for creation ...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>{\"Reasoning\": \"The question asks for a way to ...</td>\n",
+       "      <td>{\"Final Answer\": \"g\", \"Reasoning\": \"Recycling ...</td>\n",
+       "      <td>{\"Reasoning\": \"Mining, fossil fuels, deforesta...</td>\n",
+       "      <td>{\"Final Answer\": \"g\", \"Reasoning\": \"Recycling ...</td>\n",
+       "      <td>{\"Final Answer\": \"g\"}</td>\n",
+       "      <td>{\"Final Answer\": \"g\"}</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1679</th>\n",
+       "      <td>Biology</td>\n",
+       "      <td>A drought dehydrates an entire what?</td>\n",
+       "      <td>d</td>\n",
+       "      <td>a) Body water: This option is incorrect becaus...</td>\n",
+       "      <td>(a) Watershred - This is not a scientific term...</td>\n",
+       "      <td>(a) Body water (b) Dried fruit (c) Bodily wate...</td>\n",
+       "      <td>Question: A drought dehydrates an entire what?...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>{\"Reasoning\": \"The question asks for a term th...</td>\n",
+       "      <td>{\"Final Answer\": \"d\", \"Reasoning\": \"A drought ...</td>\n",
+       "      <td>{ \"Reasoning\": \"A drought is a prolonged perio...</td>\n",
+       "      <td>{\"Final Answer\": \"d\", \"Reasoning\": \"A drought ...</td>\n",
+       "      <td>{\"Final Answer\": \"d\"}</td>\n",
+       "      <td>{\"Final Answer\": \"d\"}</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1680</th>\n",
+       "      <td>Biology</td>\n",
+       "      <td>An animal requires ingestion to do what?</td>\n",
+       "      <td>e</td>\n",
+       "      <td>a) Aerobic capacity: This option is not logica...</td>\n",
+       "      <td>1. Read the question and options carefully: \"W...</td>\n",
+       "      <td>(a) Aerobic capacity (b) Die (c) Water conserv...</td>\n",
+       "      <td>Question: An animal requires ingestion to do w...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>{\"Reasoning\": \"Ingestion is the process of tak...</td>\n",
+       "      <td>{\"Final Answer\": \"e\", \"Reasoning\": \"Ingestion ...</td>\n",
+       "      <td>{\"Reasoning\": \"Ingestion is the process of tak...</td>\n",
+       "      <td>{\"Final Answer\": \"e\", \"Reasoning\": \"Ingestion ...</td>\n",
+       "      <td>{\"Final Answer\": \"d\"}</td>\n",
+       "      <td>{\"Final Answer\": \"d\"}</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1681</th>\n",
+       "      <td>Biology</td>\n",
+       "      <td>Ultraviolet light can cause what?</td>\n",
+       "      <td>b</td>\n",
+       "      <td>a) Ultraviolet light does not cause heat energ...</td>\n",
+       "      <td>1. First, read the question and options carefu...</td>\n",
+       "      <td>(a) Heat energy (b) Skin cancer (c) Killing in...</td>\n",
+       "      <td>Question: Ultraviolet light can cause what?\\nA...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>{\"Reasoning\": \"Ultraviolet (UV) light is a typ...</td>\n",
+       "      <td>{\"Final Answer\": \"b\", \"Reasoning\": \"Ultraviole...</td>\n",
+       "      <td>{\"Reasoning\": \"Ultraviolet (UV) light is a typ...</td>\n",
+       "      <td>{\"Final Answer\": \"d\", \"Reasoning\": \"Ultraviole...</td>\n",
+       "      <td>{\"Final Answer\": \"b\"}</td>\n",
+       "      <td>{\"Final Answer\": \"b\"}</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1682</th>\n",
+       "      <td>Physical activity</td>\n",
+       "      <td>What can increase a body's strength?</td>\n",
+       "      <td>c</td>\n",
+       "      <td>a) Four limbs: This option is not correct beca...</td>\n",
+       "      <td>(a) Communication: In this context, the questi...</td>\n",
+       "      <td>(a) Four limbs (b) Disease (c) Running (d) Bic...</td>\n",
+       "      <td>Question: What can increase a body's strength?...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>{\"Reasoning\": \"The question asks for something...</td>\n",
+       "      <td>{\"Final Answer\": \"c\", \"Reasoning\": \"Running is...</td>\n",
+       "      <td>{\"Reasoning\": \"A body's strength is primarily ...</td>\n",
+       "      <td>{\"Final Answer\": \"c\", \"Reasoning\": \"Running is...</td>\n",
+       "      <td>{\"Final Answer\": \"c\"}</td>\n",
+       "      <td>{\"Final Answer\": \"c\"}</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>1683 rows × 21 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                  topic                                      question_text  \\\n",
+       "0        Transportation                          What are busses used for?   \n",
+       "1        Climate change  Which of the following does not contribute to ...   \n",
+       "2           Photography  What uses electrical energy converted from che...   \n",
+       "3          Microbiology                 Bacteria causes what to be harmed?   \n",
+       "4               Biology                         Plants and snakes live _.?   \n",
+       "...                 ...                                                ...   \n",
+       "1678            Biology  New resources required for creation can be red...   \n",
+       "1679            Biology               A drought dehydrates an entire what?   \n",
+       "1680            Biology           An animal requires ingestion to do what?   \n",
+       "1681            Biology                  Ultraviolet light can cause what?   \n",
+       "1682  Physical activity               What can increase a body's strength?   \n",
+       "\n",
+       "     answer_key                                   gpt3_5_reasoning  \\\n",
+       "0             b  a) Protective shelter: This option is incorrec...   \n",
+       "1             g  a) Nucleus of a cell: This option is not relat...   \n",
+       "2             b  a) Sunlight: Sunlight is a form of energy that...   \n",
+       "3             a  Now, let's go through each option and explain ...   \n",
+       "4             a  b) Important habitats: This option is incorrec...   \n",
+       "...         ...                                                ...   \n",
+       "1678          g  a) Mining: Mining involves extracting minerals...   \n",
+       "1679          d  a) Body water: This option is incorrect becaus...   \n",
+       "1680          e  a) Aerobic capacity: This option is not logica...   \n",
+       "1681          b  a) Ultraviolet light does not cause heat energ...   \n",
+       "1682          c  a) Four limbs: This option is not correct beca...   \n",
+       "\n",
+       "                                      mistral_reasoning  \\\n",
+       "0     1. Start by reading the question carefully: \"C...   \n",
+       "1     To solve this question, let's first understand...   \n",
+       "2     1. Read the question and options carefully: Th...   \n",
+       "3     To answer this question correctly, let's follo...   \n",
+       "4     1. Read the question and options carefully: Th...   \n",
+       "...                                                 ...   \n",
+       "1678  1. Start by reading the question and options c...   \n",
+       "1679  (a) Watershred - This is not a scientific term...   \n",
+       "1680  1. Read the question and options carefully: \"W...   \n",
+       "1681  1. First, read the question and options carefu...   \n",
+       "1682  (a) Communication: In this context, the questi...   \n",
+       "\n",
+       "                                         answer_choices  \\\n",
+       "0     (a) Protective shelter (b) Transporting humans...   \n",
+       "1     (a) Nucleus of a cell (b) Flying in a plane (c...   \n",
+       "2     (a) Sunlight (b) Cameras (c) Cells (d) Buses (...   \n",
+       "3     (a) Plants (b) Electronics (c) Fossils (d) Hum...   \n",
+       "4     (a) Almost everywhere (b) Important habitats (...   \n",
+       "...                                                 ...   \n",
+       "1678  (a) Mining (b) Mutations (c) Fossil fuels (d) ...   \n",
+       "1679  (a) Body water (b) Dried fruit (c) Bodily wate...   \n",
+       "1680  (a) Aerobic capacity (b) Die (c) Water conserv...   \n",
+       "1681  (a) Heat energy (b) Skin cancer (c) Killing in...   \n",
+       "1682  (a) Four limbs (b) Disease (c) Running (d) Bic...   \n",
+       "\n",
+       "                                            user_prompt  \\\n",
+       "0     Question: What are busses used for?\\nAnswer Ch...   \n",
+       "1     Question: Which of the following does not cont...   \n",
+       "2     Question: What uses electrical energy converte...   \n",
+       "3     Question: Bacteria causes what to be harmed?\\n...   \n",
+       "4     Question: Plants and snakes live _.?\\nAnswer C...   \n",
+       "...                                                 ...   \n",
+       "1678  Question: New resources required for creation ...   \n",
+       "1679  Question: A drought dehydrates an entire what?...   \n",
+       "1680  Question: An animal requires ingestion to do w...   \n",
+       "1681  Question: Ultraviolet light can cause what?\\nA...   \n",
+       "1682  Question: What can increase a body's strength?...   \n",
+       "\n",
+       "                                        user_prompt_RFA  \\\n",
+       "0     <s>[INST] Answer the Question and include your...   \n",
+       "1     <s>[INST] Answer the Question and include your...   \n",
+       "2     <s>[INST] Answer the Question and include your...   \n",
+       "3     <s>[INST] Answer the Question and include your...   \n",
+       "4     <s>[INST] Answer the Question and include your...   \n",
+       "...                                                 ...   \n",
+       "1678  <s>[INST] Answer the Question and include your...   \n",
+       "1679  <s>[INST] Answer the Question and include your...   \n",
+       "1680  <s>[INST] Answer the Question and include your...   \n",
+       "1681  <s>[INST] Answer the Question and include your...   \n",
+       "1682  <s>[INST] Answer the Question and include your...   \n",
+       "\n",
+       "                                conversation_RFA_gpt3_5  \\\n",
+       "0     <s>[INST] Answer the Question and include your...   \n",
+       "1     <s>[INST] Answer the Question and include your...   \n",
+       "2     <s>[INST] Answer the Question and include your...   \n",
+       "3     <s>[INST] Answer the Question and include your...   \n",
+       "4     <s>[INST] Answer the Question and include your...   \n",
+       "...                                                 ...   \n",
+       "1678  <s>[INST] Answer the Question and include your...   \n",
+       "1679  <s>[INST] Answer the Question and include your...   \n",
+       "1680  <s>[INST] Answer the Question and include your...   \n",
+       "1681  <s>[INST] Answer the Question and include your...   \n",
+       "1682  <s>[INST] Answer the Question and include your...   \n",
+       "\n",
+       "                               conversation_RFA_mistral  ...  \\\n",
+       "0     <s>[INST] Answer the Question and include your...  ...   \n",
+       "1     <s>[INST] Answer the Question and include your...  ...   \n",
+       "2     <s>[INST] Answer the Question and include your...  ...   \n",
+       "3     <s>[INST] Answer the Question and include your...  ...   \n",
+       "4     <s>[INST] Answer the Question and include your...  ...   \n",
+       "...                                                 ...  ...   \n",
+       "1678  <s>[INST] Answer the Question and include your...  ...   \n",
+       "1679  <s>[INST] Answer the Question and include your...  ...   \n",
+       "1680  <s>[INST] Answer the Question and include your...  ...   \n",
+       "1681  <s>[INST] Answer the Question and include your...  ...   \n",
+       "1682  <s>[INST] Answer the Question and include your...  ...   \n",
+       "\n",
+       "                                conversation_FAR_gpt3_5  \\\n",
+       "0     <s>[INST] Answer the Question and include your...   \n",
+       "1     <s>[INST] Answer the Question and include your...   \n",
+       "2     <s>[INST] Answer the Question and include your...   \n",
+       "3     <s>[INST] Answer the Question and include your...   \n",
+       "4     <s>[INST] Answer the Question and include your...   \n",
+       "...                                                 ...   \n",
+       "1678  <s>[INST] Answer the Question and include your...   \n",
+       "1679  <s>[INST] Answer the Question and include your...   \n",
+       "1680  <s>[INST] Answer the Question and include your...   \n",
+       "1681  <s>[INST] Answer the Question and include your...   \n",
+       "1682  <s>[INST] Answer the Question and include your...   \n",
+       "\n",
+       "                               conversation_FAR_mistral  \\\n",
+       "0     <s>[INST] Answer the Question and include your...   \n",
+       "1     <s>[INST] Answer the Question and include your...   \n",
+       "2     <s>[INST] Answer the Question and include your...   \n",
+       "3     <s>[INST] Answer the Question and include your...   \n",
+       "4     <s>[INST] Answer the Question and include your...   \n",
+       "...                                                 ...   \n",
+       "1678  <s>[INST] Answer the Question and include your...   \n",
+       "1679  <s>[INST] Answer the Question and include your...   \n",
+       "1680  <s>[INST] Answer the Question and include your...   \n",
+       "1681  <s>[INST] Answer the Question and include your...   \n",
+       "1682  <s>[INST] Answer the Question and include your...   \n",
+       "\n",
+       "                                         user_prompt_FA  \\\n",
+       "0     <s>[INST] Answer the Question and include your...   \n",
+       "1     <s>[INST] Answer the Question and include your...   \n",
+       "2     <s>[INST] Answer the Question and include your...   \n",
+       "3     <s>[INST] Answer the Question and include your...   \n",
+       "4     <s>[INST] Answer the Question and include your...   \n",
+       "...                                                 ...   \n",
+       "1678  <s>[INST] Answer the Question and include your...   \n",
+       "1679  <s>[INST] Answer the Question and include your...   \n",
+       "1680  <s>[INST] Answer the Question and include your...   \n",
+       "1681  <s>[INST] Answer the Question and include your...   \n",
+       "1682  <s>[INST] Answer the Question and include your...   \n",
+       "\n",
+       "                                        conversation_FA  \\\n",
+       "0     <s>[INST] Answer the Question and include your...   \n",
+       "1     <s>[INST] Answer the Question and include your...   \n",
+       "2     <s>[INST] Answer the Question and include your...   \n",
+       "3     <s>[INST] Answer the Question and include your...   \n",
+       "4     <s>[INST] Answer the Question and include your...   \n",
+       "...                                                 ...   \n",
+       "1678  <s>[INST] Answer the Question and include your...   \n",
+       "1679  <s>[INST] Answer the Question and include your...   \n",
+       "1680  <s>[INST] Answer the Question and include your...   \n",
+       "1681  <s>[INST] Answer the Question and include your...   \n",
+       "1682  <s>[INST] Answer the Question and include your...   \n",
+       "\n",
+       "                                  responses_RFA_mistral  \\\n",
+       "0     {\"Reasoning\": \"Busses are primarily used for t...   \n",
+       "1     {\"Reasoning\": \"Global warming is primarily cau...   \n",
+       "2     {\"Reasoning\": \"The question asks for an organi...   \n",
+       "3     {\"Reasoning\": \"Bacteria are microorganisms tha...   \n",
+       "4     {\"Reasoning\": \"The question asks about the liv...   \n",
+       "...                                                 ...   \n",
+       "1678  {\"Reasoning\": \"The question asks for a way to ...   \n",
+       "1679  {\"Reasoning\": \"The question asks for a term th...   \n",
+       "1680  {\"Reasoning\": \"Ingestion is the process of tak...   \n",
+       "1681  {\"Reasoning\": \"Ultraviolet (UV) light is a typ...   \n",
+       "1682  {\"Reasoning\": \"The question asks for something...   \n",
+       "\n",
+       "                                  responses_FAR_mistral  \\\n",
+       "0     {\"Final Answer\": \"b\", \"Reasoning\": \"Busses are...   \n",
+       "1     {\"Final Answer\": \"a\", \"Reasoning\": \"The nucleu...   \n",
+       "2     {\"Final Answer\": \"c\", \"Reasoning\": \"The proces...   \n",
+       "3     {\"Final Answer\": \"d\", \"Reasoning\": \"Bacteria c...   \n",
+       "4     {\"Final Answer\": \"a\", \"Reasoning\": \"Plants and...   \n",
+       "...                                                 ...   \n",
+       "1678  {\"Final Answer\": \"g\", \"Reasoning\": \"Recycling ...   \n",
+       "1679  {\"Final Answer\": \"d\", \"Reasoning\": \"A drought ...   \n",
+       "1680  {\"Final Answer\": \"e\", \"Reasoning\": \"Ingestion ...   \n",
+       "1681  {\"Final Answer\": \"b\", \"Reasoning\": \"Ultraviole...   \n",
+       "1682  {\"Final Answer\": \"c\", \"Reasoning\": \"Running is...   \n",
+       "\n",
+       "                                   responses_RFA_gpt3_5  \\\n",
+       "0     {\"Reasoning\": \"Busses are primarily used for t...   \n",
+       "1     {\"Reasoning\": \"The nucleus of a cell (option a...   \n",
+       "2     {\"Reasoning\": \"The process of converting chemi...   \n",
+       "3     { \"Reasoning\": \"Bacteria can cause harm to var...   \n",
+       "4     {\"Reasoning\": \"The question asks about the rel...   \n",
+       "...                                                 ...   \n",
+       "1678  {\"Reasoning\": \"Mining, fossil fuels, deforesta...   \n",
+       "1679  { \"Reasoning\": \"A drought is a prolonged perio...   \n",
+       "1680  {\"Reasoning\": \"Ingestion is the process of tak...   \n",
+       "1681  {\"Reasoning\": \"Ultraviolet (UV) light is a typ...   \n",
+       "1682  {\"Reasoning\": \"A body's strength is primarily ...   \n",
+       "\n",
+       "                                   responses_FAR_gpt3_5  \\\n",
+       "0     {\"Final Answer\": \"b\", \"Reasoning\": \"Busses are...   \n",
+       "1     {\"Final Answer\": \"a\", \"Reasoning\": \"The nucleu...   \n",
+       "2     {\"Final Answer\": \"e\", \"Reasoning\": \"Bacteria u...   \n",
+       "3     {\"Final Answer\": \"d\", \"Reasoning\": \"Bacteria c...   \n",
+       "4     {\"Final Answer\": \"f\", \"Reasoning\": \"Plants and...   \n",
+       "...                                                 ...   \n",
+       "1678  {\"Final Answer\": \"g\", \"Reasoning\": \"Recycling ...   \n",
+       "1679  {\"Final Answer\": \"d\", \"Reasoning\": \"A drought ...   \n",
+       "1680  {\"Final Answer\": \"e\", \"Reasoning\": \"Ingestion ...   \n",
+       "1681  {\"Final Answer\": \"d\", \"Reasoning\": \"Ultraviole...   \n",
+       "1682  {\"Final Answer\": \"c\", \"Reasoning\": \"Running is...   \n",
+       "\n",
+       "               responses_FA         responses_base  \n",
+       "0     {\"Final Answer\": \"b\"}  {\"Final Answer\": \"b\"}  \n",
+       "1     {\"Final Answer\": \"a\"}  {\"Final Answer\": \"a\"}  \n",
+       "2     {\"Final Answer\": \"c\"}  {\"Final Answer\": \"c\"}  \n",
+       "3     {\"Final Answer\": \"d\"}  {\"Final Answer\": \"d\"}  \n",
+       "4     {\"Final Answer\": \"g\"}  {\"Final Answer\": \"g\"}  \n",
+       "...                     ...                    ...  \n",
+       "1678  {\"Final Answer\": \"g\"}  {\"Final Answer\": \"g\"}  \n",
+       "1679  {\"Final Answer\": \"d\"}  {\"Final Answer\": \"d\"}  \n",
+       "1680  {\"Final Answer\": \"d\"}  {\"Final Answer\": \"d\"}  \n",
+       "1681  {\"Final Answer\": \"b\"}  {\"Final Answer\": \"b\"}  \n",
+       "1682  {\"Final Answer\": \"c\"}  {\"Final Answer\": \"c\"}  \n",
+       "\n",
+       "[1683 rows x 21 columns]"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "8619f9f5-9fe4-433e-b524-51c2b12e8d12",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def extract_final_answer(response):\n",
+    "    return json.loads(response).get(\"Final Answer\")\n",
+    "\n",
+    "# Create new columns for predictions\n",
+    "df['predictions_base'] = df['responses_base'].apply(extract_final_answer)\n",
+    "df['predictions_FA'] = df['responses_FA'].apply(extract_final_answer)\n",
+    "df['predictions_RFA_mistral'] = df['responses_RFA_mistral'].apply(extract_final_answer)\n",
+    "df['predictions_FAR_mistral'] = df['responses_FAR_mistral'].apply(extract_final_answer)\n",
+    "df['predictions_RFA_gpt3_5'] = df['responses_RFA_gpt3_5'].apply(extract_final_answer)\n",
+    "df['predictions_FAR_gpt3_5'] = df['responses_FAR_gpt3_5'].apply(extract_final_answer)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "938cf2a3-2fed-42a3-82ec-a56cb0ea9f37",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Base: \t\t\t\t\t\t45.28%\n",
+      "Final Answer: \t\t\t\t\t45.4%\n",
+      "Reasoning and then the Final Answer (Mistral): \t53.89%\n",
+      "Final Answer and then the Reasoning (Mistral): \t60.72%\n",
+      "Reasoning and then the Final Answer (GPT-3.5): \t59.06%\n",
+      "Final Answer and then the Reasoning (GPT-3.5): \t60.31%\n"
+     ]
+    }
+   ],
+   "source": [
+    "from sklearn.metrics import accuracy_score\n",
+    "\n",
+    "print(f\"Base: \\t\\t\\t\\t\\t\\t{round(accuracy_score(y_true=df['answer_key'], y_pred=df['predictions_base']) * 100, 2)}%\")\n",
+    "print(f\"Final Answer: \\t\\t\\t\\t\\t{round(accuracy_score(y_true=df['answer_key'], y_pred=df['predictions_FA']) * 100, 2)}%\")\n",
+    "print(f\"Reasoning and then the Final Answer (Mistral): \\t{round(accuracy_score(y_true=df['answer_key'], y_pred=df['predictions_RFA_mistral']) * 100, 2)}%\")\n",
+    "print(f\"Final Answer and then the Reasoning (Mistral): \\t{round(accuracy_score(y_true=df['answer_key'], y_pred=df['predictions_FAR_mistral']) * 100, 2)}%\")\n",
+    "print(f\"Reasoning and then the Final Answer (GPT-3.5): \\t{round(accuracy_score(y_true=df['answer_key'], y_pred=df['predictions_RFA_gpt3_5']) * 100, 2)}%\")\n",
+    "print(f\"Final Answer and then the Reasoning (GPT-3.5): \\t{round(accuracy_score(y_true=df['answer_key'], y_pred=df['predictions_FAR_gpt3_5']) * 100, 2)}%\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "83aae472-513b-43c3-9ee8-64d4cda775e0",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>topic</th>\n",
+       "      <th>question_text</th>\n",
+       "      <th>answer_key</th>\n",
+       "      <th>gpt3_5_reasoning</th>\n",
+       "      <th>mistral_reasoning</th>\n",
+       "      <th>answer_choices</th>\n",
+       "      <th>user_prompt</th>\n",
+       "      <th>user_prompt_RFA</th>\n",
+       "      <th>conversation_RFA_gpt3_5</th>\n",
+       "      <th>conversation_RFA_mistral</th>\n",
+       "      <th>...</th>\n",
+       "      <th>responses_RFA_gpt3_5</th>\n",
+       "      <th>responses_FAR_gpt3_5</th>\n",
+       "      <th>responses_FA</th>\n",
+       "      <th>responses_base</th>\n",
+       "      <th>predictions_base</th>\n",
+       "      <th>predictions_FA</th>\n",
+       "      <th>predictions_RFA_mistral</th>\n",
+       "      <th>predictions_FAR_mistral</th>\n",
+       "      <th>predictions_RFA_gpt3_5</th>\n",
+       "      <th>predictions_FAR_gpt3_5</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Transportation</td>\n",
+       "      <td>What are busses used for?</td>\n",
+       "      <td>b</td>\n",
+       "      <td>a) Protective shelter: This option is incorrec...</td>\n",
+       "      <td>1. Start by reading the question carefully: \"C...</td>\n",
+       "      <td>(a) Protective shelter (b) Transporting humans...</td>\n",
+       "      <td>Question: What are busses used for?\\nAnswer Ch...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>{\"Reasoning\": \"Busses are primarily used for t...</td>\n",
+       "      <td>{\"Final Answer\": \"b\", \"Reasoning\": \"Busses are...</td>\n",
+       "      <td>{\"Final Answer\": \"b\"}</td>\n",
+       "      <td>{\"Final Answer\": \"b\"}</td>\n",
+       "      <td>b</td>\n",
+       "      <td>b</td>\n",
+       "      <td>b</td>\n",
+       "      <td>b</td>\n",
+       "      <td>b</td>\n",
+       "      <td>b</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Climate change</td>\n",
+       "      <td>Which of the following does not contribute to ...</td>\n",
+       "      <td>g</td>\n",
+       "      <td>a) Nucleus of a cell: This option is not relat...</td>\n",
+       "      <td>To solve this question, let's first understand...</td>\n",
+       "      <td>(a) Nucleus of a cell (b) Flying in a plane (c...</td>\n",
+       "      <td>Question: Which of the following does not cont...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>{\"Reasoning\": \"The nucleus of a cell (option a...</td>\n",
+       "      <td>{\"Final Answer\": \"a\", \"Reasoning\": \"The nucleu...</td>\n",
+       "      <td>{\"Final Answer\": \"a\"}</td>\n",
+       "      <td>{\"Final Answer\": \"a\"}</td>\n",
+       "      <td>a</td>\n",
+       "      <td>a</td>\n",
+       "      <td>g</td>\n",
+       "      <td>a</td>\n",
+       "      <td>a</td>\n",
+       "      <td>a</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Photography</td>\n",
+       "      <td>What uses electrical energy converted from che...</td>\n",
+       "      <td>b</td>\n",
+       "      <td>a) Sunlight: Sunlight is a form of energy that...</td>\n",
+       "      <td>1. Read the question and options carefully: Th...</td>\n",
+       "      <td>(a) Sunlight (b) Cameras (c) Cells (d) Buses (...</td>\n",
+       "      <td>Question: What uses electrical energy converte...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>{\"Reasoning\": \"The process of converting chemi...</td>\n",
+       "      <td>{\"Final Answer\": \"e\", \"Reasoning\": \"Bacteria u...</td>\n",
+       "      <td>{\"Final Answer\": \"c\"}</td>\n",
+       "      <td>{\"Final Answer\": \"c\"}</td>\n",
+       "      <td>c</td>\n",
+       "      <td>c</td>\n",
+       "      <td>e</td>\n",
+       "      <td>c</td>\n",
+       "      <td>c</td>\n",
+       "      <td>e</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Microbiology</td>\n",
+       "      <td>Bacteria causes what to be harmed?</td>\n",
+       "      <td>a</td>\n",
+       "      <td>Now, let's go through each option and explain ...</td>\n",
+       "      <td>To answer this question correctly, let's follo...</td>\n",
+       "      <td>(a) Plants (b) Electronics (c) Fossils (d) Hum...</td>\n",
+       "      <td>Question: Bacteria causes what to be harmed?\\n...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>{ \"Reasoning\": \"Bacteria can cause harm to var...</td>\n",
+       "      <td>{\"Final Answer\": \"d\", \"Reasoning\": \"Bacteria c...</td>\n",
+       "      <td>{\"Final Answer\": \"d\"}</td>\n",
+       "      <td>{\"Final Answer\": \"d\"}</td>\n",
+       "      <td>d</td>\n",
+       "      <td>d</td>\n",
+       "      <td>d</td>\n",
+       "      <td>d</td>\n",
+       "      <td>d</td>\n",
+       "      <td>d</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Biology</td>\n",
+       "      <td>Plants and snakes live _.?</td>\n",
+       "      <td>a</td>\n",
+       "      <td>b) Important habitats: This option is incorrec...</td>\n",
+       "      <td>1. Read the question and options carefully: Th...</td>\n",
+       "      <td>(a) Almost everywhere (b) Important habitats (...</td>\n",
+       "      <td>Question: Plants and snakes live _.?\\nAnswer C...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>{\"Reasoning\": \"The question asks about the rel...</td>\n",
+       "      <td>{\"Final Answer\": \"f\", \"Reasoning\": \"Plants and...</td>\n",
+       "      <td>{\"Final Answer\": \"g\"}</td>\n",
+       "      <td>{\"Final Answer\": \"g\"}</td>\n",
+       "      <td>g</td>\n",
+       "      <td>g</td>\n",
+       "      <td>b</td>\n",
+       "      <td>a</td>\n",
+       "      <td>f</td>\n",
+       "      <td>f</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1678</th>\n",
+       "      <td>Biology</td>\n",
+       "      <td>New resources required for creation can be red...</td>\n",
+       "      <td>g</td>\n",
+       "      <td>a) Mining: Mining involves extracting minerals...</td>\n",
+       "      <td>1. Start by reading the question and options c...</td>\n",
+       "      <td>(a) Mining (b) Mutations (c) Fossil fuels (d) ...</td>\n",
+       "      <td>Question: New resources required for creation ...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>{\"Reasoning\": \"Mining, fossil fuels, deforesta...</td>\n",
+       "      <td>{\"Final Answer\": \"g\", \"Reasoning\": \"Recycling ...</td>\n",
+       "      <td>{\"Final Answer\": \"g\"}</td>\n",
+       "      <td>{\"Final Answer\": \"g\"}</td>\n",
+       "      <td>g</td>\n",
+       "      <td>g</td>\n",
+       "      <td>g</td>\n",
+       "      <td>g</td>\n",
+       "      <td>g</td>\n",
+       "      <td>g</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1679</th>\n",
+       "      <td>Biology</td>\n",
+       "      <td>A drought dehydrates an entire what?</td>\n",
+       "      <td>d</td>\n",
+       "      <td>a) Body water: This option is incorrect becaus...</td>\n",
+       "      <td>(a) Watershred - This is not a scientific term...</td>\n",
+       "      <td>(a) Body water (b) Dried fruit (c) Bodily wate...</td>\n",
+       "      <td>Question: A drought dehydrates an entire what?...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>{ \"Reasoning\": \"A drought is a prolonged perio...</td>\n",
+       "      <td>{\"Final Answer\": \"d\", \"Reasoning\": \"A drought ...</td>\n",
+       "      <td>{\"Final Answer\": \"d\"}</td>\n",
+       "      <td>{\"Final Answer\": \"d\"}</td>\n",
+       "      <td>d</td>\n",
+       "      <td>d</td>\n",
+       "      <td>d</td>\n",
+       "      <td>d</td>\n",
+       "      <td>f</td>\n",
+       "      <td>d</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1680</th>\n",
+       "      <td>Biology</td>\n",
+       "      <td>An animal requires ingestion to do what?</td>\n",
+       "      <td>e</td>\n",
+       "      <td>a) Aerobic capacity: This option is not logica...</td>\n",
+       "      <td>1. Read the question and options carefully: \"W...</td>\n",
+       "      <td>(a) Aerobic capacity (b) Die (c) Water conserv...</td>\n",
+       "      <td>Question: An animal requires ingestion to do w...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>{\"Reasoning\": \"Ingestion is the process of tak...</td>\n",
+       "      <td>{\"Final Answer\": \"e\", \"Reasoning\": \"Ingestion ...</td>\n",
+       "      <td>{\"Final Answer\": \"d\"}</td>\n",
+       "      <td>{\"Final Answer\": \"d\"}</td>\n",
+       "      <td>d</td>\n",
+       "      <td>d</td>\n",
+       "      <td>e</td>\n",
+       "      <td>e</td>\n",
+       "      <td>e</td>\n",
+       "      <td>e</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1681</th>\n",
+       "      <td>Biology</td>\n",
+       "      <td>Ultraviolet light can cause what?</td>\n",
+       "      <td>b</td>\n",
+       "      <td>a) Ultraviolet light does not cause heat energ...</td>\n",
+       "      <td>1. First, read the question and options carefu...</td>\n",
+       "      <td>(a) Heat energy (b) Skin cancer (c) Killing in...</td>\n",
+       "      <td>Question: Ultraviolet light can cause what?\\nA...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>{\"Reasoning\": \"Ultraviolet (UV) light is a typ...</td>\n",
+       "      <td>{\"Final Answer\": \"d\", \"Reasoning\": \"Ultraviole...</td>\n",
+       "      <td>{\"Final Answer\": \"b\"}</td>\n",
+       "      <td>{\"Final Answer\": \"b\"}</td>\n",
+       "      <td>b</td>\n",
+       "      <td>b</td>\n",
+       "      <td>b</td>\n",
+       "      <td>b</td>\n",
+       "      <td>b</td>\n",
+       "      <td>d</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1682</th>\n",
+       "      <td>Physical activity</td>\n",
+       "      <td>What can increase a body's strength?</td>\n",
+       "      <td>c</td>\n",
+       "      <td>a) Four limbs: This option is not correct beca...</td>\n",
+       "      <td>(a) Communication: In this context, the questi...</td>\n",
+       "      <td>(a) Four limbs (b) Disease (c) Running (d) Bic...</td>\n",
+       "      <td>Question: What can increase a body's strength?...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>{\"Reasoning\": \"A body's strength is primarily ...</td>\n",
+       "      <td>{\"Final Answer\": \"c\", \"Reasoning\": \"Running is...</td>\n",
+       "      <td>{\"Final Answer\": \"c\"}</td>\n",
+       "      <td>{\"Final Answer\": \"c\"}</td>\n",
+       "      <td>c</td>\n",
+       "      <td>c</td>\n",
+       "      <td>f</td>\n",
+       "      <td>c</td>\n",
+       "      <td>f</td>\n",
+       "      <td>c</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>1683 rows × 27 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                  topic                                      question_text  \\\n",
+       "0        Transportation                          What are busses used for?   \n",
+       "1        Climate change  Which of the following does not contribute to ...   \n",
+       "2           Photography  What uses electrical energy converted from che...   \n",
+       "3          Microbiology                 Bacteria causes what to be harmed?   \n",
+       "4               Biology                         Plants and snakes live _.?   \n",
+       "...                 ...                                                ...   \n",
+       "1678            Biology  New resources required for creation can be red...   \n",
+       "1679            Biology               A drought dehydrates an entire what?   \n",
+       "1680            Biology           An animal requires ingestion to do what?   \n",
+       "1681            Biology                  Ultraviolet light can cause what?   \n",
+       "1682  Physical activity               What can increase a body's strength?   \n",
+       "\n",
+       "     answer_key                                   gpt3_5_reasoning  \\\n",
+       "0             b  a) Protective shelter: This option is incorrec...   \n",
+       "1             g  a) Nucleus of a cell: This option is not relat...   \n",
+       "2             b  a) Sunlight: Sunlight is a form of energy that...   \n",
+       "3             a  Now, let's go through each option and explain ...   \n",
+       "4             a  b) Important habitats: This option is incorrec...   \n",
+       "...         ...                                                ...   \n",
+       "1678          g  a) Mining: Mining involves extracting minerals...   \n",
+       "1679          d  a) Body water: This option is incorrect becaus...   \n",
+       "1680          e  a) Aerobic capacity: This option is not logica...   \n",
+       "1681          b  a) Ultraviolet light does not cause heat energ...   \n",
+       "1682          c  a) Four limbs: This option is not correct beca...   \n",
+       "\n",
+       "                                      mistral_reasoning  \\\n",
+       "0     1. Start by reading the question carefully: \"C...   \n",
+       "1     To solve this question, let's first understand...   \n",
+       "2     1. Read the question and options carefully: Th...   \n",
+       "3     To answer this question correctly, let's follo...   \n",
+       "4     1. Read the question and options carefully: Th...   \n",
+       "...                                                 ...   \n",
+       "1678  1. Start by reading the question and options c...   \n",
+       "1679  (a) Watershred - This is not a scientific term...   \n",
+       "1680  1. Read the question and options carefully: \"W...   \n",
+       "1681  1. First, read the question and options carefu...   \n",
+       "1682  (a) Communication: In this context, the questi...   \n",
+       "\n",
+       "                                         answer_choices  \\\n",
+       "0     (a) Protective shelter (b) Transporting humans...   \n",
+       "1     (a) Nucleus of a cell (b) Flying in a plane (c...   \n",
+       "2     (a) Sunlight (b) Cameras (c) Cells (d) Buses (...   \n",
+       "3     (a) Plants (b) Electronics (c) Fossils (d) Hum...   \n",
+       "4     (a) Almost everywhere (b) Important habitats (...   \n",
+       "...                                                 ...   \n",
+       "1678  (a) Mining (b) Mutations (c) Fossil fuels (d) ...   \n",
+       "1679  (a) Body water (b) Dried fruit (c) Bodily wate...   \n",
+       "1680  (a) Aerobic capacity (b) Die (c) Water conserv...   \n",
+       "1681  (a) Heat energy (b) Skin cancer (c) Killing in...   \n",
+       "1682  (a) Four limbs (b) Disease (c) Running (d) Bic...   \n",
+       "\n",
+       "                                            user_prompt  \\\n",
+       "0     Question: What are busses used for?\\nAnswer Ch...   \n",
+       "1     Question: Which of the following does not cont...   \n",
+       "2     Question: What uses electrical energy converte...   \n",
+       "3     Question: Bacteria causes what to be harmed?\\n...   \n",
+       "4     Question: Plants and snakes live _.?\\nAnswer C...   \n",
+       "...                                                 ...   \n",
+       "1678  Question: New resources required for creation ...   \n",
+       "1679  Question: A drought dehydrates an entire what?...   \n",
+       "1680  Question: An animal requires ingestion to do w...   \n",
+       "1681  Question: Ultraviolet light can cause what?\\nA...   \n",
+       "1682  Question: What can increase a body's strength?...   \n",
+       "\n",
+       "                                        user_prompt_RFA  \\\n",
+       "0     <s>[INST] Answer the Question and include your...   \n",
+       "1     <s>[INST] Answer the Question and include your...   \n",
+       "2     <s>[INST] Answer the Question and include your...   \n",
+       "3     <s>[INST] Answer the Question and include your...   \n",
+       "4     <s>[INST] Answer the Question and include your...   \n",
+       "...                                                 ...   \n",
+       "1678  <s>[INST] Answer the Question and include your...   \n",
+       "1679  <s>[INST] Answer the Question and include your...   \n",
+       "1680  <s>[INST] Answer the Question and include your...   \n",
+       "1681  <s>[INST] Answer the Question and include your...   \n",
+       "1682  <s>[INST] Answer the Question and include your...   \n",
+       "\n",
+       "                                conversation_RFA_gpt3_5  \\\n",
+       "0     <s>[INST] Answer the Question and include your...   \n",
+       "1     <s>[INST] Answer the Question and include your...   \n",
+       "2     <s>[INST] Answer the Question and include your...   \n",
+       "3     <s>[INST] Answer the Question and include your...   \n",
+       "4     <s>[INST] Answer the Question and include your...   \n",
+       "...                                                 ...   \n",
+       "1678  <s>[INST] Answer the Question and include your...   \n",
+       "1679  <s>[INST] Answer the Question and include your...   \n",
+       "1680  <s>[INST] Answer the Question and include your...   \n",
+       "1681  <s>[INST] Answer the Question and include your...   \n",
+       "1682  <s>[INST] Answer the Question and include your...   \n",
+       "\n",
+       "                               conversation_RFA_mistral  ...  \\\n",
+       "0     <s>[INST] Answer the Question and include your...  ...   \n",
+       "1     <s>[INST] Answer the Question and include your...  ...   \n",
+       "2     <s>[INST] Answer the Question and include your...  ...   \n",
+       "3     <s>[INST] Answer the Question and include your...  ...   \n",
+       "4     <s>[INST] Answer the Question and include your...  ...   \n",
+       "...                                                 ...  ...   \n",
+       "1678  <s>[INST] Answer the Question and include your...  ...   \n",
+       "1679  <s>[INST] Answer the Question and include your...  ...   \n",
+       "1680  <s>[INST] Answer the Question and include your...  ...   \n",
+       "1681  <s>[INST] Answer the Question and include your...  ...   \n",
+       "1682  <s>[INST] Answer the Question and include your...  ...   \n",
+       "\n",
+       "                                   responses_RFA_gpt3_5  \\\n",
+       "0     {\"Reasoning\": \"Busses are primarily used for t...   \n",
+       "1     {\"Reasoning\": \"The nucleus of a cell (option a...   \n",
+       "2     {\"Reasoning\": \"The process of converting chemi...   \n",
+       "3     { \"Reasoning\": \"Bacteria can cause harm to var...   \n",
+       "4     {\"Reasoning\": \"The question asks about the rel...   \n",
+       "...                                                 ...   \n",
+       "1678  {\"Reasoning\": \"Mining, fossil fuels, deforesta...   \n",
+       "1679  { \"Reasoning\": \"A drought is a prolonged perio...   \n",
+       "1680  {\"Reasoning\": \"Ingestion is the process of tak...   \n",
+       "1681  {\"Reasoning\": \"Ultraviolet (UV) light is a typ...   \n",
+       "1682  {\"Reasoning\": \"A body's strength is primarily ...   \n",
+       "\n",
+       "                                   responses_FAR_gpt3_5  \\\n",
+       "0     {\"Final Answer\": \"b\", \"Reasoning\": \"Busses are...   \n",
+       "1     {\"Final Answer\": \"a\", \"Reasoning\": \"The nucleu...   \n",
+       "2     {\"Final Answer\": \"e\", \"Reasoning\": \"Bacteria u...   \n",
+       "3     {\"Final Answer\": \"d\", \"Reasoning\": \"Bacteria c...   \n",
+       "4     {\"Final Answer\": \"f\", \"Reasoning\": \"Plants and...   \n",
+       "...                                                 ...   \n",
+       "1678  {\"Final Answer\": \"g\", \"Reasoning\": \"Recycling ...   \n",
+       "1679  {\"Final Answer\": \"d\", \"Reasoning\": \"A drought ...   \n",
+       "1680  {\"Final Answer\": \"e\", \"Reasoning\": \"Ingestion ...   \n",
+       "1681  {\"Final Answer\": \"d\", \"Reasoning\": \"Ultraviole...   \n",
+       "1682  {\"Final Answer\": \"c\", \"Reasoning\": \"Running is...   \n",
+       "\n",
+       "               responses_FA         responses_base predictions_base  \\\n",
+       "0     {\"Final Answer\": \"b\"}  {\"Final Answer\": \"b\"}                b   \n",
+       "1     {\"Final Answer\": \"a\"}  {\"Final Answer\": \"a\"}                a   \n",
+       "2     {\"Final Answer\": \"c\"}  {\"Final Answer\": \"c\"}                c   \n",
+       "3     {\"Final Answer\": \"d\"}  {\"Final Answer\": \"d\"}                d   \n",
+       "4     {\"Final Answer\": \"g\"}  {\"Final Answer\": \"g\"}                g   \n",
+       "...                     ...                    ...              ...   \n",
+       "1678  {\"Final Answer\": \"g\"}  {\"Final Answer\": \"g\"}                g   \n",
+       "1679  {\"Final Answer\": \"d\"}  {\"Final Answer\": \"d\"}                d   \n",
+       "1680  {\"Final Answer\": \"d\"}  {\"Final Answer\": \"d\"}                d   \n",
+       "1681  {\"Final Answer\": \"b\"}  {\"Final Answer\": \"b\"}                b   \n",
+       "1682  {\"Final Answer\": \"c\"}  {\"Final Answer\": \"c\"}                c   \n",
+       "\n",
+       "     predictions_FA predictions_RFA_mistral predictions_FAR_mistral  \\\n",
+       "0                 b                       b                       b   \n",
+       "1                 a                       g                       a   \n",
+       "2                 c                       e                       c   \n",
+       "3                 d                       d                       d   \n",
+       "4                 g                       b                       a   \n",
+       "...             ...                     ...                     ...   \n",
+       "1678              g                       g                       g   \n",
+       "1679              d                       d                       d   \n",
+       "1680              d                       e                       e   \n",
+       "1681              b                       b                       b   \n",
+       "1682              c                       f                       c   \n",
+       "\n",
+       "     predictions_RFA_gpt3_5 predictions_FAR_gpt3_5  \n",
+       "0                         b                      b  \n",
+       "1                         a                      a  \n",
+       "2                         c                      e  \n",
+       "3                         d                      d  \n",
+       "4                         f                      f  \n",
+       "...                     ...                    ...  \n",
+       "1678                      g                      g  \n",
+       "1679                      f                      d  \n",
+       "1680                      e                      e  \n",
+       "1681                      b                      d  \n",
+       "1682                      f                      c  \n",
+       "\n",
+       "[1683 rows x 27 columns]"
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "45c08dd4-0b98-4e0f-b487-549f60518a4e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "30c9a9f7656f4950a3fde9deaa6bf0ac",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "bfc21892a7eb45caa097435b6026087d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Creating parquet from Arrow format:   0%|          | 0/2 [00:00<?, ?ba/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "CommitInfo(commit_url='https://huggingface.co/datasets/derek-thomas/labeled-multiple-choice-explained-mistral-results/commit/b2c01c5867b23afe06c0806f2886152864574946', commit_message='Upload dataset', commit_description='', oid='b2c01c5867b23afe06c0806f2886152864574946', pr_url=None, repo_url=RepoUrl('https://huggingface.co/datasets/derek-thomas/labeled-multiple-choice-explained-mistral-results', endpoint='https://huggingface.co', repo_type='dataset', repo_id='derek-thomas/labeled-multiple-choice-explained-mistral-results'), pr_revision=None, pr_num=None)"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from datasets import Dataset, DatasetDict\n",
+    "\n",
+    "# Create dataset from df\n",
+    "df.reset_index(drop=True, inplace=True)\n",
+    "dataset = Dataset.from_pandas(df)\n",
+    "\n",
+    "# Push dataset to the hub\n",
+    "dataset.push_to_hub(OUTPUT_DATASET)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "45ed7bf8-54de-4f4d-aab5-b78af8f54ccc",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

autotrain/autotrain.sh

@@ -0,0 +1,28 @@
+#EXPERIMENT_TYPES=("R-FA" "FA-R" "FA")
+EXPERIMENT_TYPES=("FA")
+
+for EXPERIMENT_TYPE in ${EXPERIMENT_TYPES[@]}; do
+  echo "Running autotrain for experiment type: ${EXPERIMENT_TYPE}"
+
+  # Export the variable to make it available to envsubst
+  export EXPERIMENT_TYPE=${EXPERIMENT_TYPE}
+
+  # Substitute the variable into the template
+  envsubst < ./autotrain/autotrain_template.yml > ./autotrain/autotrain.yml
+
+  # Echo the contents of the substituted YAML file
+  echo "Generated autotrain.yml:"
+  cat ./autotrain/autotrain.yml
+  echo "-----------------------------------"
+
+  # Run the autotrain command
+  autotrain --config ./autotrain/autotrain.yml
+
+  # Check if the command was successful
+  if [[ $? -ne 0 ]]; then
+    echo "Error running autotrain for ${EXPERIMENT_TYPE}, exiting."
+    exit 1
+  fi
+done
+
+echo "All experiments completed successfully!"

autotrain/autotrain.yml

@@ -0,0 +1,34 @@
+task: llm-sft
+base_model: mistralai/Mistral-7B-Instruct-v0.3
+project_name: autotrain-mistral-v03-prompt-experiment-mc-R-FA-sg
+log: tensorboard
+backend: spaces-l4x1
+
+data:
+  path: derek-thomas/labeled-multiple-choice-explained-mistral-tokenized
+  train_split: train
+#   valid_split: val
+  valid_split: null
+  chat_template: none
+  column_mapping:
+    text_column: conversation_R_FA_sg
+
+params:
+  block_size: 1024
+  model_max_length: 1024
+  epochs: 2
+  batch_size: 1
+  lr: 3e-5
+  peft: true
+  quantization: int4
+  target_modules: all-linear
+  padding: left
+  optimizer: adamw_torch
+  scheduler: linear
+  gradient_accumulation: 8
+  mixed_precision: bf16
+
+hub:
+  username: derek-thomas
+  token: ${HF_TOKEN}
+  push_to_hub: true

autotrain/run_autotrain.py

@@ -0,0 +1,67 @@
+import os
+import subprocess
+
+import yaml
+
+# Base config
+config_template = {
+    "task": "llm-sft",
+    "base_model": "mistralai/Mistral-7B-Instruct-v0.3",
+    "project_name": "",
+    "log": "tensorboard",
+    "backend": "spaces-l4x1",
+    "data": {
+        "path": "derek-thomas/labeled-multiple-choice-explained-mistral-tokenized",
+        "train_split": "train",
+        "valid_split": None,
+        "chat_template": "none",
+        "column_mapping": {
+            "text_column": ""
+            },
+        },
+    "params": {
+        "block_size": 1024,
+        "model_max_length": 1024,
+        "epochs": 2,
+        "batch_size": 1,
+        "lr": 3e-5,
+        "peft": True,
+        "quantization": "int4",
+        "target_modules": "all-linear",
+        "padding": "left",
+        "optimizer": "adamw_torch",
+        "scheduler": "linear",
+        "gradient_accumulation": 8,
+        "mixed_precision": "bf16",
+        },
+    "hub": {
+        "username": "derek-thomas",
+        "token": os.getenv('HF_TOKEN'),
+        "push_to_hub": True,
+        },
+    }
+
+# Suffix options
+project_suffixes = ["RFA-gpt3-5", "RFA-mistral", "FAR-gpt3-5", "FAR-mistral", "FA"]
+text_columns = ["conversation_RFA_gpt3_5", "conversation_RFA_mistral", "conversation_FAR_gpt3_5",
+                "conversation_FAR_mistral", "conversation_FA"]
+
+# Directory to store generated configs
+output_dir = "./autotrain_configs"
+os.makedirs(output_dir, exist_ok=True)
+
+# Generate configs and run commands
+for project_suffix, text_column in zip(project_suffixes, text_columns):
+    # Modify the config
+    config = config_template.copy()
+    config["project_name"] = f"mistral-v03-poe-{project_suffix}"
+    config["data"]["column_mapping"]["text_column"] = text_column
+
+    # Save the config to a YAML file
+    config_path = os.path.join(output_dir, f"{text_column}.yml")
+    with open(config_path, "w") as f:
+        yaml.dump(config, f)
+
+    # Run the command
+    print(f"Running autotrain with config: {config_path}")
+    subprocess.run(["autotrain", "--config", config_path])

requirements.txt

@@ -0,0 +1,11 @@
+autotrain-advanced
+wandb
+huggingface_hub
+datasets
+pandas
+gradio
+scikit-learn
+lighteval[tensorboardX,adapters]
+nest_asyncio
+plotly
+ipywidgets

utilities/clean_up_spaces.ipynb

@@ -0,0 +1,180 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "291ab1c2-0ff0-41bf-ba56-e0c5ff2cde92",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "AUTHOR = 'your-user-name'\n",
+    "SEARCH_STRING = 'autotrain-autotrain-mistral'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "1c68cc85-bed7-4270-8745-87f6fc7cb8d0",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "2bfbe27ca548439a860e624431f1c90d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from huggingface_hub import login\n",
+    "\n",
+    "login()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "7f96de54-43ac-4bd4-a65f-15d4046914c7",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Deleting:\tderek-thomas/autotrain-autotrain-mistral-v03-prompt-experiment-mc-FA\n",
+      "Deleting:\tderek-thomas/autotrain-autotrain-mistral-v03-prompt-experiment-mc-FA-R\n",
+      "Deleting:\tderek-thomas/autotrain-autotrain-mistral-v03-prompt-experiment-mc-R-FA\n",
+      "Deleting:\tderek-thomas/autotrain-autotrain-mistral-v03-prompt-experiment-mc-FA-sg\n",
+      "Deleting:\tderek-thomas/autotrain-autotrain-mistral-v03-prompt-experiment-mc-FA-R-sg\n",
+      "Deleting:\tderek-thomas/autotrain-autotrain-mistral-v03-prompt-experiment-mc-R-FA-sg\n"
+     ]
+    }
+   ],
+   "source": [
+    "from huggingface_hub import list_spaces, delete_repo\n",
+    "\n",
+    "# Retrieve your spaces\n",
+    "spaces = list_spaces(author=AUTHOR)\n",
+    "\n",
+    "# Print space names\n",
+    "for space in spaces:\n",
+    "    if SEARCH_STRING in space.id:\n",
+    "        delete_repo(repo_id=space.id, repo_type=\"space\")\n",
+    "        print(f\"Deleting:\\t{space.id}\")\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "a9f439dd-e8d6-44d8-b68c-eca1f6ae40ff",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "derek-thomas/dreambooth-training-hannah\n",
+      "derek-thomas/dreambooth-training-hannah-v21-v2\n",
+      "derek-thomas/derek-thomas-dreambooth-training-hannah-v15-v3\n",
+      "derek-thomas/dreambooth-training-hannah-v15-v4\n",
+      "derek-thomas/dreambooth-training-hannah-v15-v4-1672074451\n",
+      "derek-thomas/dreambooth-training-hannah-v15-v5\n",
+      "derek-thomas/dreambooth-training-hannah-v15-v6\n",
+      "derek-thomas/xlm-roberta-base-finetuned-panx-de\n",
+      "derek-thomas/lal-parser\n",
+      "derek-thomas/naughtyformer\n",
+      "derek-thomas/ibert-quantized\n",
+      "derek-thomas/t5-end2end-question-generation\n",
+      "derek-thomas/distilhubert-finetuned-gtzan\n",
+      "derek-thomas/distilhubert-finetuned-gtzan-efficient\n",
+      "derek-thomas/Hubert_emotion-finetuned-gtzan-efficient\n",
+      "derek-thomas/hubert-large-ll60k-finetuned-gtzan-efficient\n",
+      "derek-thomas/hubert-base-ls960-finetuned-gtzan-efficient\n",
+      "derek-thomas/hubert-base-ls960-finetuned-gtzan-efficient-label-smoothed\n",
+      "derek-thomas/whisper-tiny-polyai-english\n",
+      "derek-thomas/speecht5_tts-finetuned_voxpopuli_lt\n",
+      "derek-thomas/jais-13b-chat-hf\n",
+      "derek-thomas/sd-class-butterflies-32\n",
+      "derek-thomas/speecht5_tts-finetuned_voxpopuli_hr\n",
+      "derek-thomas/speecht5_finetuned_voxpopuli_hr\n",
+      "derek-thomas/tgi-benchmark-notebooks\n",
+      "derek-thomas/Meta-Llama-3-8B-Instruct\n",
+      "derek-thomas/tgi-notebooks-optimization\n",
+      "derek-thomas/trained-flux\n",
+      "derek-thomas/hannah-dreambooth\n",
+      "derek-thomas/hannah-dreambooth-2\n",
+      "derek-thomas/hannah-book-creation\n",
+      "derek-thomas/hannah-dreambooth-3\n",
+      "derek-thomas/hannah-dreambooth-4\n",
+      "derek-thomas/bert-ner-openvino\n",
+      "derek-thomas/hannah-dreambooth-5\n",
+      "derek-thomas/prompt-experiment-sft-multiple-choice\n",
+      "derek-thomas/autotrain-mistral-v03-prompt-experiment-mc-FA\n",
+      "Deleting:\tderek-thomas/autotrain-mistral-v03-prompt-experiment-mc-FA\n",
+      "derek-thomas/autotrain-mistral-v03-prompt-experiment-mc-R-FA-sg\n",
+      "Deleting:\tderek-thomas/autotrain-mistral-v03-prompt-experiment-mc-R-FA-sg\n",
+      "derek-thomas/autotrain-mistral-v03-prompt-experiment-mc-FA-R\n",
+      "Deleting:\tderek-thomas/autotrain-mistral-v03-prompt-experiment-mc-FA-R\n",
+      "derek-thomas/autotrain-mistral-v03-prompt-experiment-mc-FA-sg\n",
+      "Deleting:\tderek-thomas/autotrain-mistral-v03-prompt-experiment-mc-FA-sg\n",
+      "derek-thomas/autotrain-mistral-v03-prompt-experiment-mc-R-FA\n",
+      "Deleting:\tderek-thomas/autotrain-mistral-v03-prompt-experiment-mc-R-FA\n",
+      "derek-thomas/autotrain-mistral-v03-prompt-experiment-mc-FA-R-sg\n",
+      "Deleting:\tderek-thomas/autotrain-mistral-v03-prompt-experiment-mc-FA-R-sg\n"
+     ]
+    }
+   ],
+   "source": [
+    "from huggingface_hub import list_models\n",
+    "\n",
+    "# Retrieve your spaces\n",
+    "models = list_models(author=AUTHOR)\n",
+    "\n",
+    "# Print space names\n",
+    "for model in models:\n",
+    "    # print(model.id)\n",
+    "    if 'autotrain-mistral' in model.id:\n",
+    "        delete_repo(repo_id=model.id, repo_type=\"model\")\n",
+    "        print(f\"Deleting:\\t{model.id}\")\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8064d700-e880-4022-9c91-55ef60c493cd",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}