{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [], "gpuType": "T4" }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" }, "accelerator": "GPU", "widgets": { "application/vnd.jupyter.widget-state+json": { "1830ab16750b4c7ebf5d1692a02e3544": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_de65aa83f07a48a98ecc54fd956c5831", "IPY_MODEL_93dad1c7c99b4a36a8df85fc2dff617e", "IPY_MODEL_098a2232a1f34d94ab4f61141733730f" ], "layout": "IPY_MODEL_79a38d122e4c4b2593b96151f5bce47e" } }, "de65aa83f07a48a98ecc54fd956c5831": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_d8e51c8c5d0b4698918e24b3aa32f492", "placeholder": "​", "style": "IPY_MODEL_b8d38b2204b94010acdd81a438c6c58c", "value": "Loading checkpoint shards: 100%" } }, "93dad1c7c99b4a36a8df85fc2dff617e": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_1ac8c0d157144c48bae5444f35076192", "max": 3, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_818b9628fa4f4781922add05931a8503", "value": 3 } }, "098a2232a1f34d94ab4f61141733730f": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_920b4172ad2044eda1cdedb2af7c9ed7", "placeholder": "​", "style": "IPY_MODEL_8da0941661e7430d9dbb1984e19cb0ed", "value": " 3/3 [01:03<00:00, 20.58s/it]" } }, "79a38d122e4c4b2593b96151f5bce47e": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "d8e51c8c5d0b4698918e24b3aa32f492": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "b8d38b2204b94010acdd81a438c6c58c": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "1ac8c0d157144c48bae5444f35076192": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "818b9628fa4f4781922add05931a8503": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "920b4172ad2044eda1cdedb2af7c9ed7": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "8da0941661e7430d9dbb1984e19cb0ed": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } } } } }, "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "id": "ege5kkFDUiEf" }, "outputs": [], "source": [ "!pip -q install transformers\n", "!pip -q install sentencepiece\n", "!pip -q install accelerate" ] }, { "cell_type": "code", "source": [ "import torch\n", "from transformers import LlamaForCausalLM, LlamaTokenizer\n", "\n", "# Hugging Face model_path\n", "model_path = 'psmathur/orca_mini_3b'\n", "tokenizer = LlamaTokenizer.from_pretrained(model_path)\n", "model = LlamaForCausalLM.from_pretrained(\n", " model_path, torch_dtype=torch.float16, device_map='auto',\n", ")\n", "\n", "\n", "#generate text function\n", "def generate_text(system, instruction, input=None):\n", "\n", " if input:\n", " prompt = f\"### System:\\n{system}\\n\\n### User:\\n{instruction}\\n\\n### Input:\\n{input}\\n\\n### Response:\\n\"\n", " else:\n", " prompt = f\"### System:\\n{system}\\n\\n### User:\\n{instruction}\\n\\n### Response:\\n\"\n", "\n", " tokens = tokenizer.encode(prompt)\n", " tokens = torch.LongTensor(tokens).unsqueeze(0)\n", " tokens = tokens.to('cuda')\n", "\n", " instance = {'input_ids': tokens,'top_p': 1.0, 'temperature':0.7, 'generate_len': 1024, 'top_k': 50}\n", "\n", " length = len(tokens[0])\n", " with torch.no_grad():\n", " rest = model.generate(\n", " input_ids=tokens,\n", " max_length=length+instance['generate_len'],\n", " use_cache=True,\n", " do_sample=True,\n", " top_p=instance['top_p'],\n", " temperature=instance['temperature'],\n", " top_k=instance['top_k']\n", " )\n", " output = rest[0][length:]\n", " string = tokenizer.decode(output, skip_special_tokens=True)\n", " return f'[!] Response: {string}'\n", "\n", "# Sample Test Instruction Used by Youtuber Sam Witteveen https://www.youtube.com/@samwitteveenai\n", "system = 'You are an AI assistant that follows instruction extremely well. Help as much as you can.'\n", "instruction = 'Write a letter to Sam Altman, CEO of OpenAI, requesting him to convert GPT4 a private model by OpenAI to an open source project'\n", "print(generate_text(system, instruction))" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 319, "referenced_widgets": [ "1830ab16750b4c7ebf5d1692a02e3544", "de65aa83f07a48a98ecc54fd956c5831", "93dad1c7c99b4a36a8df85fc2dff617e", "098a2232a1f34d94ab4f61141733730f", "79a38d122e4c4b2593b96151f5bce47e", "d8e51c8c5d0b4698918e24b3aa32f492", "b8d38b2204b94010acdd81a438c6c58c", "1ac8c0d157144c48bae5444f35076192", "818b9628fa4f4781922add05931a8503", "920b4172ad2044eda1cdedb2af7c9ed7", "8da0941661e7430d9dbb1984e19cb0ed" ] }, "id": "c8FJEmZgUxqo", "outputId": "ceca1b51-dd4d-4d61-bec9-42dd6f074cde" }, "execution_count": 2, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "WARNING:accelerate.utils.modeling:The model weights are not tied. Please use the `tie_weights` method before using the `infer_auto_device` function.\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ "Loading checkpoint shards: 0%| | 0/3 [00:00