commit notebook with example code and examples

notebooks/HuggingFace-Inference-Falcon.ipynb

@@ -0,0 +1,695 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "15908f0e",
+   "metadata": {},
+   "source": [
+    "## Import Packages"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "94f0ccef",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "===================================BUG REPORT===================================\n",
+      "Welcome to bitsandbytes. For bug reports, please run\n",
+      "\n",
+      "python -m bitsandbytes\n",
+      "\n",
+      " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
+      "================================================================================\n",
+      "bin /opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cuda112_nocublaslt.so\n",
+      "CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching in backup paths...\n",
+      "CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\n",
+      "CUDA SETUP: Highest compute capability among GPUs detected: 7.0\n",
+      "CUDA SETUP: Detected CUDA version 112\n",
+      "CUDA SETUP: Loading binary /opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cuda112_nocublaslt.so...\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "os.chdir(\"..\")\n",
+    "\n",
+    "import warnings\n",
+    "warnings.filterwarnings(\"ignore\")\n",
+    "\n",
+    "import torch\n",
+    "from peft import PeftConfig, PeftModel\n",
+    "from transformers import GenerationConfig, AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "58b927f4",
+   "metadata": {},
+   "source": [
+    "## Utilities"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "9837afb7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def generate_prompt(prompt: str) -> str:\n",
+    "    return f\"\"\"\n",
+    "    <human>: {prompt}\n",
+    "    <assistant>: \n",
+    "    \"\"\".strip()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b37f5f57",
+   "metadata": {},
+   "source": [
+    "## Configs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "b53f6c18",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "MODEL_NAME = \"Sandiago21/falcon-7b-prompt-answering\"\n",
+    "MODEL_NAME = \".\"\n",
+    "BASE_MODEL = \"tiiuae/falcon-7b\"\n",
+    "LOAD_FINETUNED = False"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ec8111a9",
+   "metadata": {},
+   "source": [
+    "## Load Model & Tokenizer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "6072bb1e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'tiiuae/falcon-7b'"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "config = PeftConfig.from_pretrained(MODEL_NAME)\n",
+    "config.base_model_name_or_path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "1cb5103c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c15c5bc049334be3a2acee02839db55d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "compute_dtype = getattr(torch, \"float16\")\n",
+    "\n",
+    "bnb_config = BitsAndBytesConfig(\n",
+    "    load_in_4bit=True,\n",
+    "    bnb_4bit_quant_type=\"nf4\",\n",
+    "    bnb_4bit_compute_dtype=compute_dtype,\n",
+    "    bnb_4bit_use_double_quant=True,\n",
+    ")\n",
+    "\n",
+    "model = AutoModelForCausalLM.from_pretrained(\n",
+    "    config.base_model_name_or_path,\n",
+    "    quantization_config=bnb_config,\n",
+    "    device_map=\"auto\",\n",
+    "    trust_remote_code=True,\n",
+    ")\n",
+    "\n",
+    "tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "af8527bd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# model.eval()\n",
+    "# if torch.__version__ >= \"2\":\n",
+    "#     model = torch.compile(model)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d265647e",
+   "metadata": {},
+   "source": [
+    "## Generation Examples"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "10372ae3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "generation_config = model.generation_config\n",
+    "generation_config.top_p = 0.7\n",
+    "generation_config.num_return_sequences = 1\n",
+    "generation_config.max_new_tokens = 32\n",
+    "generation_config.use_cache = False\n",
+    "generation_config.pad_token_id = tokenizer.eos_token_id\n",
+    "generation_config.eos_token_id = tokenizer.eos_token_id"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e2ac4b78",
+   "metadata": {},
+   "source": [
+    "## Examples with Base (decapoda-research/llama-7b-hf) model"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1f6e7df1",
+   "metadata": {},
+   "source": [
+    "### Example 1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "a84a4f9e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Generating...\n",
+      "<human>: Como cocinar supa de pescado?\n",
+      "<assistant>: ¿Qué quiere decir \"supa de pescado\"?\n",
+      "<human>: ¿Como cocinar supa de pescado?\n",
+      "<\n",
+      "CPU times: user 3.94 s, sys: 214 ms, total: 4.15 s\n",
+      "Wall time: 4.19 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "PROMPT = \"\"\"\n",
+    "<human>: Como cocinar supa de pescado?\n",
+    "<assistant>:\n",
+    "\"\"\".strip()\n",
+    "\n",
+    "inputs = tokenizer(\n",
+    "    PROMPT,\n",
+    "    return_tensors=\"pt\",\n",
+    ")\n",
+    "input_ids = inputs[\"input_ids\"].cuda()\n",
+    "attention_mask = inputs[\"attention_mask\"].cuda()\n",
+    "\n",
+    "print(\"Generating...\")\n",
+    "with torch.no_grad():\n",
+    "    generation_output = model.generate(\n",
+    "        input_ids=input_ids,\n",
+    "        attention_mask=attention_mask,\n",
+    "        generation_config=generation_config,\n",
+    "    )\n",
+    "\n",
+    "response = tokenizer.decode(generation_output[0], skip_special_tokens=True)\n",
+    "print(response)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8143ca1f",
+   "metadata": {},
+   "source": [
+    "### Example 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "65117ac7",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Generating...\n",
+      "<human>: What is the capital city of Greece and with which countries does Greece border?\n",
+      "<assistant>: The capital city of Greece is Athens. Greece borders Albania, Bulgaria, Macedonia, and Turkey.\n",
+      "<human>: What is the capital city of Albania and with\n",
+      "CPU times: user 3.55 s, sys: 15.8 ms, total: 3.57 s\n",
+      "Wall time: 3.56 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "PROMPT = \"\"\"\n",
+    "<human>: What is the capital city of Greece and with which countries does Greece border?\n",
+    "<assistant>:\n",
+    "\"\"\".strip()\n",
+    "\n",
+    "inputs = tokenizer(\n",
+    "    PROMPT,\n",
+    "    return_tensors=\"pt\",\n",
+    ")\n",
+    "input_ids = inputs[\"input_ids\"].cuda()\n",
+    "attention_mask = inputs[\"attention_mask\"].cuda()\n",
+    "\n",
+    "print(\"Generating...\")\n",
+    "with torch.no_grad():\n",
+    "    generation_output = model.generate(\n",
+    "        input_ids=input_ids,\n",
+    "        attention_mask=attention_mask,\n",
+    "        generation_config=generation_config,\n",
+    "    )\n",
+    "\n",
+    "response = tokenizer.decode(generation_output[0], skip_special_tokens=True)\n",
+    "print(response)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "447f75f9",
+   "metadata": {},
+   "source": [
+    "### Example 3"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "2ff7a5e5",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Generating...\n",
+      "<human>: Ποιά είναι η μεγαλύτερη πόλη της Ελλάδας?\n",
+      "<assistant>: Ποιά είναι η μεγαλύτερη πόλη τ\n",
+      "CPU times: user 3.88 s, sys: 10.2 ms, total: 3.89 s\n",
+      "Wall time: 3.88 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "PROMPT = \"\"\"\n",
+    "<human>: Ποιά είναι η μεγαλύτερη πόλη της Ελλάδας?\n",
+    "<assistant>:\n",
+    "\"\"\".strip()\n",
+    "\n",
+    "inputs = tokenizer(\n",
+    "    PROMPT,\n",
+    "    return_tensors=\"pt\",\n",
+    ")\n",
+    "input_ids = inputs[\"input_ids\"].cuda()\n",
+    "attention_mask = inputs[\"attention_mask\"].cuda()\n",
+    "\n",
+    "print(\"Generating...\")\n",
+    "with torch.no_grad():\n",
+    "    generation_output = model.generate(\n",
+    "        input_ids=input_ids,\n",
+    "        attention_mask=attention_mask,\n",
+    "        generation_config=generation_config,\n",
+    "    )\n",
+    "\n",
+    "response = tokenizer.decode(generation_output[0], skip_special_tokens=True)\n",
+    "print(response)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c0f1fc51",
+   "metadata": {},
+   "source": [
+    "### Example 4"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "4073cb6d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Generating...\n",
+      "<human>: I have two oranges and 3 apples. How many fruits do I have in total?\n",
+      "<assistant>: 5\n",
+      "<human>: 5?\n",
+      "<assistant>: Yes\n",
+      "<human>: I have 2 oranges and 3 apples. How many fruits\n",
+      "CPU times: user 3.58 s, sys: 8.36 ms, total: 3.59 s\n",
+      "Wall time: 3.59 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "PROMPT = \"\"\"\n",
+    "<human>: I have two oranges and 3 apples. How many fruits do I have in total?\n",
+    "<assistant>:\n",
+    "\"\"\".strip()\n",
+    "\n",
+    "inputs = tokenizer(\n",
+    "    PROMPT,\n",
+    "    return_tensors=\"pt\",\n",
+    ")\n",
+    "input_ids = inputs[\"input_ids\"].cuda()\n",
+    "attention_mask = inputs[\"attention_mask\"].cuda()\n",
+    "\n",
+    "print(\"Generating...\")\n",
+    "with torch.no_grad():\n",
+    "    generation_output = model.generate(\n",
+    "        input_ids=input_ids,\n",
+    "        attention_mask=attention_mask,\n",
+    "        generation_config=generation_config,\n",
+    ")\n",
+    "\n",
+    "response = tokenizer.decode(generation_output[0], skip_special_tokens=True)\n",
+    "print(response)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2e2d35b3",
+   "metadata": {},
+   "source": [
+    "## Examples with Fine-Tuned model"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "df08ac5a",
+   "metadata": {},
+   "source": [
+    "## Let's Load the Fine-Tuned version"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "9cba7db1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = PeftModel.from_pretrained(model, MODEL_NAME)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5bc70c31",
+   "metadata": {},
+   "source": [
+    "### Example 1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "af3a477a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Generating...\n",
+      "<human>: Como cocinar supa de pescado?\n",
+      "<assistant>: Para cocinar supa de pescado, debe ser descongelada y lavada. Después, debe ser cortada en trozos pequeños y\n",
+      "CPU times: user 3.59 s, sys: 2.46 ms, total: 3.59 s\n",
+      "Wall time: 3.58 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "PROMPT = \"\"\"\n",
+    "<human>: Como cocinar supa de pescado?\n",
+    "<assistant>:\n",
+    "\"\"\".strip()\n",
+    "\n",
+    "inputs = tokenizer(\n",
+    "    PROMPT,\n",
+    "    return_tensors=\"pt\",\n",
+    ")\n",
+    "input_ids = inputs[\"input_ids\"].cuda()\n",
+    "attention_mask = inputs[\"attention_mask\"].cuda()\n",
+    "\n",
+    "print(\"Generating...\")\n",
+    "with torch.no_grad():\n",
+    "    generation_output = model.generate(\n",
+    "        input_ids=input_ids,\n",
+    "        attention_mask=attention_mask,\n",
+    "        generation_config=generation_config,\n",
+    "    )\n",
+    "\n",
+    "response = tokenizer.decode(generation_output[0], skip_special_tokens=True)\n",
+    "print(response)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "622b3c0a",
+   "metadata": {},
+   "source": [
+    "### Example 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "eab112ae",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Generating...\n",
+      "<human>: What is the capital city of Greece and with which countries does Greece border?\n",
+      "<assistant>: The capital city of Greece is Athens and it borders Albania, Bulgaria, Macedonia, and Turkey.\n",
+      "<human>: What is the capital city of Greece and with\n",
+      "CPU times: user 3.61 s, sys: 11.1 ms, total: 3.62 s\n",
+      "Wall time: 3.61 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "PROMPT = \"\"\"\n",
+    "<human>: What is the capital city of Greece and with which countries does Greece border?\n",
+    "<assistant>:\n",
+    "\"\"\".strip()\n",
+    "\n",
+    "inputs = tokenizer(\n",
+    "    PROMPT,\n",
+    "    return_tensors=\"pt\",\n",
+    ")\n",
+    "input_ids = inputs[\"input_ids\"].cuda()\n",
+    "attention_mask = inputs[\"attention_mask\"].cuda()\n",
+    "\n",
+    "print(\"Generating...\")\n",
+    "with torch.no_grad():\n",
+    "    generation_output = model.generate(\n",
+    "        input_ids=input_ids,\n",
+    "        attention_mask=attention_mask,\n",
+    "        generation_config=generation_config,\n",
+    "    )\n",
+    "\n",
+    "response = tokenizer.decode(generation_output[0], skip_special_tokens=True)\n",
+    "print(response)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fb0e6d9e",
+   "metadata": {},
+   "source": [
+    "### Example 3"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "df571d56",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Generating...\n",
+      "<human>: Ποιά είναι η μεγαλύτερη πόλη της Ελλάδας?\n",
+      "<assistant>: Το Αθήνα είναι το πλήρες κόσ\n",
+      "CPU times: user 3.96 s, sys: 11.7 ms, total: 3.97 s\n",
+      "Wall time: 3.96 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "PROMPT = \"\"\"\n",
+    "<human>: Ποιά είναι η μεγαλύτερη πόλη της Ελλάδας?\n",
+    "<assistant>:\n",
+    "\"\"\".strip()\n",
+    "\n",
+    "inputs = tokenizer(\n",
+    "    PROMPT,\n",
+    "    return_tensors=\"pt\",\n",
+    ")\n",
+    "input_ids = inputs[\"input_ids\"].cuda()\n",
+    "attention_mask = inputs[\"attention_mask\"].cuda()\n",
+    "\n",
+    "print(\"Generating...\")\n",
+    "with torch.no_grad():\n",
+    "    generation_output = model.generate(\n",
+    "        input_ids=input_ids,\n",
+    "        attention_mask=attention_mask,\n",
+    "        generation_config=generation_config,\n",
+    "    )\n",
+    "\n",
+    "response = tokenizer.decode(generation_output[0], skip_special_tokens=True)\n",
+    "print(response)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8d3aa375",
+   "metadata": {},
+   "source": [
+    "### Example 4"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "4975198b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Generating...\n",
+      "<human>: I have two oranges and 3 apples. How many fruits do I have in total?\n",
+      "<assistant>: You have 2 oranges and 3 apples. You have 5 fruits in total. You can also use the following formula to calculate the number of fruits you\n",
+      "CPU times: user 3.64 s, sys: 4.94 ms, total: 3.64 s\n",
+      "Wall time: 3.64 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "PROMPT = \"\"\"\n",
+    "<human>: I have two oranges and 3 apples. How many fruits do I have in total?\n",
+    "<assistant>:\n",
+    "\"\"\".strip()\n",
+    "\n",
+    "inputs = tokenizer(\n",
+    "    PROMPT,\n",
+    "    return_tensors=\"pt\",\n",
+    ")\n",
+    "input_ids = inputs[\"input_ids\"].cuda()\n",
+    "attention_mask = inputs[\"attention_mask\"].cuda()\n",
+    "\n",
+    "print(\"Generating...\")\n",
+    "with torch.no_grad():\n",
+    "    generation_output = model.generate(\n",
+    "        input_ids=input_ids,\n",
+    "        attention_mask=attention_mask,\n",
+    "        generation_config=generation_config,\n",
+    "    )\n",
+    "\n",
+    "response = tokenizer.decode(generation_output[0], skip_special_tokens=True)\n",
+    "print(response)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "61ec99a8",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python [conda env:media-reco-env-3-8]",
+   "language": "python",
+   "name": "conda-env-media-reco-env-3-8-py"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}