{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "b3b92bc7-d105-405f-970d-804d298b9976", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting accelerate\n", " Downloading accelerate-0.27.2-py3-none-any.whl.metadata (18 kB)\n", "Collecting transformers\n", " Downloading transformers-4.37.2-py3-none-any.whl.metadata (129 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.4/129.4 kB\u001b[0m \u001b[31m65.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting einops\n", " Downloading einops-0.7.0-py3-none-any.whl.metadata (13 kB)\n", "Collecting datasets\n", " Downloading datasets-2.17.1-py3-none-any.whl.metadata (20 kB)\n", "Collecting peft\n", " Downloading peft-0.8.2-py3-none-any.whl.metadata (25 kB)\n", "Collecting bitsandbytes\n", " Downloading bitsandbytes-0.42.0-py3-none-any.whl.metadata (9.9 kB)\n", "Collecting torch\n", " Downloading torch-2.2.0-cp311-cp311-manylinux1_x86_64.whl.metadata (25 kB)\n", "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.11/site-packages (from accelerate) (1.26.4)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/site-packages (from accelerate) (23.2)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.11/site-packages (from accelerate) (5.9.8)\n", "Requirement already satisfied: pyyaml in /usr/local/lib/python3.11/site-packages (from accelerate) (6.0.1)\n", "Collecting huggingface-hub (from accelerate)\n", " Downloading huggingface_hub-0.20.3-py3-none-any.whl.metadata (12 kB)\n", "Collecting safetensors>=0.3.1 (from accelerate)\n", " Downloading safetensors-0.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.8 kB)\n", "Collecting filelock (from transformers)\n", " Downloading filelock-3.13.1-py3-none-any.whl.metadata (2.8 kB)\n", "Collecting regex!=2019.12.17 (from transformers)\n", " Downloading regex-2023.12.25-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (40 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.9/40.9 kB\u001b[0m \u001b[31m450.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.11/site-packages (from transformers) (2.31.0)\n", "Collecting tokenizers<0.19,>=0.14 (from transformers)\n", " Downloading tokenizers-0.15.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n", "Collecting tqdm>=4.27 (from transformers)\n", " Downloading tqdm-4.66.2-py3-none-any.whl.metadata (57 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.6/57.6 kB\u001b[0m \u001b[31m130.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting pyarrow>=12.0.0 (from datasets)\n", " Downloading pyarrow-15.0.0-cp311-cp311-manylinux_2_28_x86_64.whl.metadata (3.0 kB)\n", "Collecting pyarrow-hotfix (from datasets)\n", " Downloading pyarrow_hotfix-0.6-py3-none-any.whl.metadata (3.6 kB)\n", "Collecting dill<0.3.9,>=0.3.0 (from datasets)\n", " Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n", "Collecting pandas (from datasets)\n", " Downloading pandas-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (19 kB)\n", "Collecting xxhash (from datasets)\n", " Downloading xxhash-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n", "Collecting multiprocess (from datasets)\n", " Downloading multiprocess-0.70.16-py311-none-any.whl.metadata (7.2 kB)\n", "Collecting fsspec<=2023.10.0,>=2023.1.0 (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets)\n", " Downloading fsspec-2023.10.0-py3-none-any.whl.metadata (6.8 kB)\n", "Requirement already satisfied: aiohttp in /usr/local/lib/python3.11/site-packages (from datasets) (3.8.3)\n", "Collecting scipy (from bitsandbytes)\n", " Downloading scipy-1.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (60 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.4/60.4 kB\u001b[0m \u001b[31m433.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.11/site-packages (from torch) (4.9.0)\n", "Collecting sympy (from torch)\n", " Downloading sympy-1.12-py3-none-any.whl.metadata (12 kB)\n", "Collecting networkx (from torch)\n", " Downloading networkx-3.2.1-py3-none-any.whl.metadata (5.2 kB)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/site-packages (from torch) (3.1.3)\n", "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch)\n", " Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch)\n", " Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch)\n", " Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n", "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch)\n", " Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n", "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch)\n", " Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch)\n", " Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Collecting nvidia-curand-cu12==10.3.2.106 (from torch)\n", " Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch)\n", " Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n", "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch)\n", " Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n", "Collecting nvidia-nccl-cu12==2.19.3 (from torch)\n", " Downloading nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl.metadata (1.8 kB)\n", "Collecting nvidia-nvtx-cu12==12.1.105 (from torch)\n", " Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.7 kB)\n", "Collecting triton==2.2.0 (from torch)\n", " Downloading triton-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.4 kB)\n", "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch)\n", " Downloading nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (23.2.0)\n", "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (2.1.1)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (6.0.5)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (4.0.3)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (1.9.4)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (1.4.1)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (1.3.1)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2.2.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2024.2.2)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/site-packages (from jinja2->torch) (2.1.5)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/site-packages (from pandas->datasets) (2.8.2)\n", "Collecting pytz>=2020.1 (from pandas->datasets)\n", " Downloading pytz-2024.1-py2.py3-none-any.whl.metadata (22 kB)\n", "Collecting tzdata>=2022.7 (from pandas->datasets)\n", " Downloading tzdata-2024.1-py2.py3-none-any.whl.metadata (1.4 kB)\n", "Collecting mpmath>=0.19 (from sympy->torch)\n", " Downloading mpmath-1.3.0-py3-none-any.whl.metadata (8.6 kB)\n", "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n", "Downloading accelerate-0.27.2-py3-none-any.whl (279 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m280.0/280.0 kB\u001b[0m \u001b[31m147.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading transformers-4.37.2-py3-none-any.whl (8.4 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.4/8.4 MB\u001b[0m \u001b[31m311.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading einops-0.7.0-py3-none-any.whl (44 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.6/44.6 kB\u001b[0m \u001b[31m441.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading datasets-2.17.1-py3-none-any.whl (536 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m536.7/536.7 kB\u001b[0m \u001b[31m504.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading peft-0.8.2-py3-none-any.whl (183 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m183.4/183.4 kB\u001b[0m \u001b[31m438.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading bitsandbytes-0.42.0-py3-none-any.whl (105.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m105.0/105.0 MB\u001b[0m \u001b[31m274.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading torch-2.2.0-cp311-cp311-manylinux1_x86_64.whl (755.5 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m755.5/755.5 MB\u001b[0m \u001b[31m235.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m255.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m220.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m200.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m566.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m272.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m277.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m280.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m303.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m315.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m166.0/166.0 MB\u001b[0m \u001b[31m258.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m194.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading triton-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (167.9 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m167.9/167.9 MB\u001b[0m \u001b[31m222.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading dill-0.3.8-py3-none-any.whl (116 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m147.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading fsspec-2023.10.0-py3-none-any.whl (166 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m166.4/166.4 kB\u001b[0m \u001b[31m221.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading huggingface_hub-0.20.3-py3-none-any.whl (330 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m330.1/330.1 kB\u001b[0m \u001b[31m248.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading pyarrow-15.0.0-cp311-cp311-manylinux_2_28_x86_64.whl (38.3 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m38.3/38.3 MB\u001b[0m \u001b[31m295.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading regex-2023.12.25-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (785 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m785.1/785.1 kB\u001b[0m \u001b[31m452.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading safetensors-0.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m456.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading tokenizers-0.15.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.6/3.6 MB\u001b[0m \u001b[31m371.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading tqdm-4.66.2-py3-none-any.whl (78 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.3/78.3 kB\u001b[0m \u001b[31m181.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading filelock-3.13.1-py3-none-any.whl (11 kB)\n", "Downloading multiprocess-0.70.16-py311-none-any.whl (143 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.5/143.5 kB\u001b[0m \u001b[31m368.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading networkx-3.2.1-py3-none-any.whl (1.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m411.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading pandas-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (13.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.0/13.0 MB\u001b[0m \u001b[31m310.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading pyarrow_hotfix-0.6-py3-none-any.whl (7.9 kB)\n", "Downloading scipy-1.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (38.4 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m38.4/38.4 MB\u001b[0m \u001b[31m195.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading sympy-1.12-py3-none-any.whl (5.7 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.7/5.7 MB\u001b[0m \u001b[31m308.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading xxhash-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.8/194.8 kB\u001b[0m \u001b[31m280.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading mpmath-1.3.0-py3-none-any.whl (536 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m536.2/536.2 kB\u001b[0m \u001b[31m399.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading pytz-2024.1-py2.py3-none-any.whl (505 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m505.5/505.5 kB\u001b[0m \u001b[31m493.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading tzdata-2024.1-py2.py3-none-any.whl (345 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m345.4/345.4 kB\u001b[0m \u001b[31m327.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl (20.5 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m20.5/20.5 MB\u001b[0m \u001b[31m280.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hInstalling collected packages: pytz, mpmath, xxhash, tzdata, tqdm, sympy, scipy, safetensors, regex, pyarrow-hotfix, pyarrow, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, networkx, fsspec, filelock, einops, dill, triton, pandas, nvidia-cusparse-cu12, nvidia-cudnn-cu12, multiprocess, huggingface-hub, bitsandbytes, tokenizers, nvidia-cusolver-cu12, transformers, torch, datasets, accelerate, peft\n", "Successfully installed accelerate-0.27.2 bitsandbytes-0.42.0 datasets-2.17.1 dill-0.3.8 einops-0.7.0 filelock-3.13.1 fsspec-2023.10.0 huggingface-hub-0.20.3 mpmath-1.3.0 multiprocess-0.70.16 networkx-3.2.1 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.3.101 nvidia-nvtx-cu12-12.1.105 pandas-2.2.0 peft-0.8.2 pyarrow-15.0.0 pyarrow-hotfix-0.6 pytz-2024.1 regex-2023.12.25 safetensors-0.4.2 scipy-1.12.0 sympy-1.12 tokenizers-0.15.2 torch-2.2.0 tqdm-4.66.2 transformers-4.37.2 triton-2.2.0 tzdata-2024.1 xxhash-3.4.1\n" ] } ], "source": [ "!pip install accelerate transformers einops datasets peft bitsandbytes torch" ] }, { "cell_type": "code", "execution_count": 2, "id": "46a53303-b585-4b02-956f-4af173410e25", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] } ], "source": [ "import torch\n", "from datasets import load_dataset, Dataset\n", "from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, Trainer, DataCollatorForLanguageModeling, BitsAndBytesConfig\n", "from peft import LoraConfig, get_peft_model\n", "import os" ] }, { "cell_type": "code", "execution_count": 3, "id": "ffa3ee11-3929-4032-8758-322cda3a912e", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "tokenizer_config.json: 100%|██████████| 7.34k/7.34k [00:00<00:00, 23.2MB/s]\n", "vocab.json: 100%|██████████| 798k/798k [00:00<00:00, 28.9MB/s]\n", "merges.txt: 100%|██████████| 456k/456k [00:00<00:00, 32.2MB/s]\n", "tokenizer.json: 100%|██████████| 2.11M/2.11M [00:00<00:00, 31.8MB/s]\n", "added_tokens.json: 100%|██████████| 1.08k/1.08k [00:00<00:00, 10.3MB/s]\n", "special_tokens_map.json: 100%|██████████| 99.0/99.0 [00:00<00:00, 543kB/s]\n", "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] } ], "source": [ "tokenizer = AutoTokenizer.from_pretrained(\"microsoft/phi-2\", trust_remote_code=True)\n", "tokenizer.pad_token = tokenizer.eos_token" ] }, { "cell_type": "code", "execution_count": 4, "id": "b17096e6-c633-4d88-8cfc-fb738c1e4ca0", "metadata": { "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "config.json: 100%|██████████| 863/863 [00:00<00:00, 6.02MB/s]\n", "configuration_phi.py: 100%|██████████| 9.26k/9.26k [00:00<00:00, 33.0MB/s]\n", "A new version of the following files was downloaded from https://huggingface.co/microsoft/phi-2:\n", "- configuration_phi.py\n", ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n", "modeling_phi.py: 100%|██████████| 62.7k/62.7k [00:00<00:00, 114MB/s]\n", "A new version of the following files was downloaded from https://huggingface.co/microsoft/phi-2:\n", "- modeling_phi.py\n", ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n", "model.safetensors.index.json: 100%|██████████| 35.7k/35.7k [00:00<00:00, 52.1MB/s]\n", "Downloading shards: 0%| | 0/2 [00:00, ?it/s]\n", "model-00001-of-00002.safetensors: 0%| | 0.00/5.00G [00:00, ?B/s]\u001b[A\n", "model-00001-of-00002.safetensors: 1%| | 52.4M/5.00G [00:00<00:10, 486MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 2%|▏ | 115M/5.00G [00:00<00:09, 535MB/s] \u001b[A\n", "model-00001-of-00002.safetensors: 4%|▍ | 189M/5.00G [00:00<00:08, 589MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 5%|▌ | 262M/5.00G [00:00<00:08, 590MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 7%|▋ | 325M/5.00G [00:00<00:08, 535MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 8%|▊ | 388M/5.00G [00:01<00:35, 131MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 9%|▉ | 451M/5.00G [00:01<00:26, 174MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 10%|█ | 503M/5.00G [00:04<01:12, 61.6MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 11%|█ | 545M/5.00G [00:05<01:25, 52.3MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 12%|█▏ | 598M/5.00G [00:06<01:28, 49.8MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 13%|█▎ | 640M/5.00G [00:06<01:07, 64.1MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 14%|█▎ | 682M/5.00G [00:07<01:20, 53.3MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 14%|█▍ | 724M/5.00G [00:07<01:00, 70.1MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 16%|█▌ | 797M/5.00G [00:08<00:38, 110MB/s] \u001b[A\n", "model-00001-of-00002.safetensors: 17%|█▋ | 839M/5.00G [00:09<00:57, 71.8MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 18%|█▊ | 891M/5.00G [00:10<01:07, 60.9MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 19%|█▊ | 933M/5.00G [00:10<00:53, 76.6MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 20%|█▉ | 975M/5.00G [00:11<01:09, 58.1MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 20%|██ | 1.02G/5.00G [00:11<00:52, 76.5MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 21%|██ | 1.05G/5.00G [00:13<01:13, 53.6MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 22%|██▏ | 1.08G/5.00G [00:14<01:30, 43.4MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 22%|██▏ | 1.12G/5.00G [00:14<01:04, 60.5MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 24%|██▍ | 1.20G/5.00G [00:14<00:37, 101MB/s] \u001b[A\n", "model-00001-of-00002.safetensors: 25%|██▌ | 1.27G/5.00G [00:14<00:24, 151MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 27%|██▋ | 1.34G/5.00G [00:14<00:17, 209MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 28%|██▊ | 1.42G/5.00G [00:14<00:13, 271MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 30%|██▉ | 1.49G/5.00G [00:14<00:10, 334MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 31%|███ | 1.55G/5.00G [00:14<00:08, 384MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 32%|███▏ | 1.61G/5.00G [00:15<00:07, 431MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 34%|███▎ | 1.68G/5.00G [00:15<00:06, 475MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 35%|███▌ | 1.75G/5.00G [00:15<00:06, 517MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 37%|███▋ | 1.82G/5.00G [00:15<00:05, 560MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 38%|███▊ | 1.90G/5.00G [00:15<00:05, 581MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 39%|███▉ | 1.97G/5.00G [00:15<00:05, 602MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 41%|████ | 2.04G/5.00G [00:15<00:04, 622MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 42%|████▏ | 2.12G/5.00G [00:15<00:04, 632MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 44%|████▍ | 2.19G/5.00G [00:15<00:04, 625MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 45%|████▌ | 2.26G/5.00G [00:16<00:04, 622MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 47%|████▋ | 2.34G/5.00G [00:16<00:04, 630MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 48%|████▊ | 2.41G/5.00G [00:16<00:04, 642MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 50%|████▉ | 2.49G/5.00G [00:16<00:03, 637MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 51%|█████ | 2.56G/5.00G [00:16<00:03, 644MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 53%|█████▎ | 2.63G/5.00G [00:16<00:03, 644MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 54%|█████▍ | 2.71G/5.00G [00:16<00:03, 653MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 56%|█████▌ | 2.78G/5.00G [00:16<00:03, 642MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 57%|█████▋ | 2.85G/5.00G [00:16<00:03, 650MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 59%|█████▊ | 2.93G/5.00G [00:17<00:03, 626MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 60%|█████▉ | 2.99G/5.00G [00:17<00:03, 618MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 61%|██████ | 3.05G/5.00G [00:17<00:03, 616MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 63%|██████▎ | 3.12G/5.00G [00:17<00:02, 630MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 64%|██████▍ | 3.20G/5.00G [00:17<00:02, 618MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 65%|██████▌ | 3.27G/5.00G [00:17<00:02, 620MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 67%|██████▋ | 3.34G/5.00G [00:17<00:02, 627MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 68%|██████▊ | 3.41G/5.00G [00:17<00:02, 575MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 69%|██████▉ | 3.47G/5.00G [00:17<00:02, 585MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 71%|███████ | 3.54G/5.00G [00:18<00:02, 599MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 72%|███████▏ | 3.61G/5.00G [00:18<00:02, 604MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 73%|███████▎ | 3.67G/5.00G [00:18<00:02, 576MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 75%|███████▍ | 3.74G/5.00G [00:18<00:02, 602MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 76%|███████▌ | 3.81G/5.00G [00:18<00:01, 600MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 78%|███████▊ | 3.88G/5.00G [00:18<00:01, 628MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 79%|███████▉ | 3.95G/5.00G [00:18<00:01, 630MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 81%|████████ | 4.03G/5.00G [00:18<00:01, 641MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 82%|████████▏ | 4.10G/5.00G [00:19<00:01, 617MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 83%|████████▎ | 4.16G/5.00G [00:19<00:01, 610MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 85%|████████▍ | 4.23G/5.00G [00:19<00:01, 614MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 86%|████████▌ | 4.29G/5.00G [00:19<00:01, 590MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 87%|████████▋ | 4.35G/5.00G [00:19<00:01, 584MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 89%|████████▊ | 4.42G/5.00G [00:19<00:00, 612MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 90%|█████████ | 4.50G/5.00G [00:19<00:00, 636MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 92%|█████████▏| 4.57G/5.00G [00:19<00:00, 620MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 93%|█████████▎| 4.65G/5.00G [00:19<00:00, 626MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 94%|█████████▍| 4.72G/5.00G [00:20<00:00, 639MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 96%|█████████▌| 4.79G/5.00G [00:20<00:00, 628MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 97%|█████████▋| 4.87G/5.00G [00:20<00:00, 609MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 99%|█████████▊| 4.93G/5.00G [00:20<00:00, 614MB/s]\u001b[A\n", "model-00001-of-00002.safetensors: 100%|██████████| 5.00G/5.00G [00:20<00:00, 244MB/s]\u001b[A\n", "Downloading shards: 50%|█████ | 1/2 [00:20<00:20, 20.52s/it]\n", "model-00002-of-00002.safetensors: 0%| | 0.00/564M [00:00, ?B/s]\u001b[A\n", "model-00002-of-00002.safetensors: 4%|▎ | 21.0M/564M [00:00<00:06, 89.4MB/s]\u001b[A\n", "model-00002-of-00002.safetensors: 17%|█▋ | 94.4M/564M [00:00<00:01, 317MB/s] \u001b[A\n", "model-00002-of-00002.safetensors: 28%|██▊ | 157M/564M [00:00<00:00, 421MB/s] \u001b[A\n", "model-00002-of-00002.safetensors: 41%|████ | 231M/564M [00:00<00:00, 502MB/s]\u001b[A\n", "model-00002-of-00002.safetensors: 54%|█████▍ | 304M/564M [00:00<00:00, 557MB/s]\u001b[A\n", "model-00002-of-00002.safetensors: 67%|██████▋ | 377M/564M [00:00<00:00, 599MB/s]\u001b[A\n", "model-00002-of-00002.safetensors: 80%|███████▉ | 451M/564M [00:00<00:00, 610MB/s]\u001b[A\n", "model-00002-of-00002.safetensors: 100%|██████████| 564M/564M [00:01<00:00, 529MB/s]\u001b[A\n", "Downloading shards: 100%|██████████| 2/2 [00:21<00:00, 10.81s/it]\n", "Loading checkpoint shards: 100%|██████████| 2/2 [00:03<00:00, 1.72s/it]\n", "generation_config.json: 100%|██████████| 124/124 [00:00<00:00, 951kB/s]\n" ] } ], "source": [ "bnb_config = BitsAndBytesConfig(\n", " load_in_4bit=True,\n", " bnb_4bit_use_double_quant=True,\n", " bnb_4bit_quant_type=\"nf4\",\n", " bnb_4bit_compute_dtype=torch.float16\n", ")\n", "\n", "model = AutoModelForCausalLM.from_pretrained(\n", " \"microsoft/phi-2\",\n", " device_map={\"\":0},\n", " trust_remote_code=True,\n", " quantization_config=bnb_config\n", ")" ] }, { "cell_type": "code", "execution_count": 5, "id": "e15aa794-e17c-4b09-a64a-c60377259218", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "PhiForCausalLM(\n", " (model): PhiModel(\n", " (embed_tokens): Embedding(51200, 2560)\n", " (embed_dropout): Dropout(p=0.0, inplace=False)\n", " (layers): ModuleList(\n", " (0-31): 32 x PhiDecoderLayer(\n", " (self_attn): PhiAttention(\n", " (q_proj): Linear4bit(in_features=2560, out_features=2560, bias=True)\n", " (k_proj): Linear4bit(in_features=2560, out_features=2560, bias=True)\n", " (v_proj): Linear4bit(in_features=2560, out_features=2560, bias=True)\n", " (dense): Linear4bit(in_features=2560, out_features=2560, bias=True)\n", " (rotary_emb): PhiRotaryEmbedding()\n", " )\n", " (mlp): PhiMLP(\n", " (activation_fn): NewGELUActivation()\n", " (fc1): Linear4bit(in_features=2560, out_features=10240, bias=True)\n", " (fc2): Linear4bit(in_features=10240, out_features=2560, bias=True)\n", " )\n", " (input_layernorm): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n", " (resid_dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (final_layernorm): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n", " )\n", " (lm_head): Linear(in_features=2560, out_features=51200, bias=True)\n", ")\n" ] } ], "source": [ "print(model)" ] }, { "cell_type": "code", "execution_count": 18, "id": "18d5599f-992d-4d8e-a90c-4d43774be473", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "trainable params: 18,350,080 || all params: 2,798,033,920 || trainable%: 0.6558204984162593\n" ] } ], "source": [ "config = LoraConfig(\n", " r=16,\n", " lora_alpha=16,\n", " #target_modules=[\"q_proj\",\"k_proj\",\"v_proj\"],\n", " lora_dropout=0.05,\n", " bias=\"none\",\n", " task_type=\"CAUSAL_LM\"\n", ")\n", "\n", "model = get_peft_model(model, config)\n", "model.print_trainable_parameters()" ] }, { "cell_type": "code", "execution_count": 19, "id": "baeee903-3dce-48b2-93c3-7a697d8c6daf", "metadata": {}, "outputs": [], "source": [ "def tokenize(sample):\n", " model_inps = tokenizer(sample[\"text\"], padding=True, truncation=True, max_length=512)\n", " return model_inps" ] }, { "cell_type": "code", "execution_count": 8, "id": "28a9b24a-a822-4fcb-96b3-d77b7ea30a5f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting scikit-learn\n", " Downloading scikit_learn-1.4.1.post1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (11 kB)\n", "Requirement already satisfied: numpy<2.0,>=1.19.5 in /usr/local/lib/python3.11/site-packages (from scikit-learn) (1.26.4)\n", "Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.11/site-packages (from scikit-learn) (1.12.0)\n", "Collecting joblib>=1.2.0 (from scikit-learn)\n", " Downloading joblib-1.3.2-py3-none-any.whl.metadata (5.4 kB)\n", "Collecting threadpoolctl>=2.0.0 (from scikit-learn)\n", " Downloading threadpoolctl-3.3.0-py3-none-any.whl.metadata (13 kB)\n", "Downloading scikit_learn-1.4.1.post1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.1 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.1/12.1 MB\u001b[0m \u001b[31m307.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading joblib-1.3.2-py3-none-any.whl (302 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m302.2/302.2 kB\u001b[0m \u001b[31m487.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading threadpoolctl-3.3.0-py3-none-any.whl (17 kB)\n", "Installing collected packages: threadpoolctl, joblib, scikit-learn\n", "Successfully installed joblib-1.3.2 scikit-learn-1.4.1.post1 threadpoolctl-3.3.0\n" ] } ], "source": [ "!pip install scikit-learn" ] }, { "cell_type": "code", "execution_count": 9, "id": "1ee7fd2a-38e4-4f23-a978-0bdeeda64d8b", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "\n", "from sklearn.model_selection import train_test_split\n", "dataset_name='data.csv'\n", "df = pd.read_csv(dataset_name)\n", "#train, test = train_test_split(df, test_size=0.2)" ] }, { "cell_type": "code", "execution_count": 10, "id": "e84c29e2-843e-42c2-8c0f-324d392e671c", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Tokenizing data: 100%|██████████| 16412/16412 [00:02<00:00, 6086.52 examples/s]\n" ] }, { "data": { "text/plain": [ "Dataset({\n", " features: ['input_ids', 'attention_mask'],\n", " num_rows: 16412\n", "})" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data_df = df\n", "data_df[\"text\"] = data_df[[\"user\", \"assistant\"]].apply(lambda x: \"question: \" + str(x[\"user\"]) + \" answer: \" + str(x[\"assistant\"]), axis=1)\n", "data = Dataset.from_pandas(data_df)\n", "tokenized_data = data.map(tokenize, batched=True, desc=\"Tokenizing data\", remove_columns=data.column_names)\n", "tokenized_data" ] }, { "cell_type": "code", "execution_count": 20, "id": "ac968254-5338-49df-950d-222b82647407", "metadata": {}, "outputs": [], "source": [ "training_arguments = TrainingArguments(\n", " output_dir=\".\",\n", " per_device_train_batch_size=4,\n", " gradient_accumulation_steps=1,\n", " learning_rate=2e-4,\n", " lr_scheduler_type=\"cosine\",\n", " save_strategy=\"epoch\",\n", " logging_steps=50,\n", " max_steps=10000,\n", " num_train_epochs=3,\n", " push_to_hub=True\n", " )" ] }, { "cell_type": "code", "execution_count": 12, "id": "729df2d0-0890-4ac4-adf3-c167a6e9669d", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: huggingface_hub in /usr/local/lib/python3.11/site-packages (0.20.3)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (3.13.1)\n", "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (2023.10.0)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (2.31.0)\n", "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (4.66.2)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (6.0.1)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (4.9.0)\n", "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (23.2)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/site-packages (from requests->huggingface_hub) (2.1.1)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/site-packages (from requests->huggingface_hub) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/site-packages (from requests->huggingface_hub) (2.2.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/site-packages (from requests->huggingface_hub) (2024.2.2)\n" ] } ], "source": [ "!pip install huggingface_hub" ] }, { "cell_type": "code", "execution_count": 13, "id": "525f81a4-eb92-466e-bb9c-cd63122231ab", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: jupyterlab in /usr/local/lib/python3.11/site-packages (4.1.0)\n", "Collecting jupyterlab\n", " Downloading jupyterlab-4.1.2-py3-none-any.whl.metadata (15 kB)\n", "Collecting jupyterlab-git\n", " Downloading jupyterlab_git-0.50.0-py3-none-any.whl.metadata (31 kB)\n", "Requirement already satisfied: async-lru>=1.0.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (2.0.4)\n", "Requirement already satisfied: httpx>=0.25.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (0.26.0)\n", "Requirement already satisfied: ipykernel in /usr/local/lib/python3.11/site-packages (from jupyterlab) (6.29.1)\n", "Requirement already satisfied: jinja2>=3.0.3 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (3.1.3)\n", "Requirement already satisfied: jupyter-core in /usr/local/lib/python3.11/site-packages (from jupyterlab) (5.7.1)\n", "Requirement already satisfied: jupyter-lsp>=2.0.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (2.2.2)\n", "Requirement already satisfied: jupyter-server<3,>=2.4.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (2.12.5)\n", "Requirement already satisfied: jupyterlab-server<3,>=2.19.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (2.25.2)\n", "Requirement already satisfied: notebook-shim>=0.2 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (0.2.3)\n", "Requirement already satisfied: packaging in /usr/local/lib/python3.11/site-packages (from jupyterlab) (23.2)\n", "Requirement already satisfied: tornado>=6.2.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (6.4)\n", "Requirement already satisfied: traitlets in /usr/local/lib/python3.11/site-packages (from jupyterlab) (5.14.1)\n", "Collecting nbdime~=4.0.1 (from jupyterlab-git)\n", " Downloading nbdime-4.0.1-py3-none-any.whl.metadata (9.5 kB)\n", "Requirement already satisfied: nbformat in /usr/local/lib/python3.11/site-packages (from jupyterlab-git) (5.9.2)\n", "Requirement already satisfied: pexpect in /usr/local/lib/python3.11/site-packages (from jupyterlab-git) (4.9.0)\n", "Requirement already satisfied: anyio in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (4.2.0)\n", "Requirement already satisfied: certifi in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (2024.2.2)\n", "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (1.0.2)\n", "Requirement already satisfied: idna in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (3.6)\n", "Requirement already satisfied: sniffio in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (1.3.0)\n", "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.11/site-packages (from httpcore==1.*->httpx>=0.25.0->jupyterlab) (0.14.0)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/site-packages (from jinja2>=3.0.3->jupyterlab) (2.1.5)\n", "Requirement already satisfied: argon2-cffi in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (23.1.0)\n", "Requirement already satisfied: jupyter-client>=7.4.4 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (8.6.0)\n", "Requirement already satisfied: jupyter-events>=0.9.0 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.9.0)\n", "Requirement already satisfied: jupyter-server-terminals in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.5.2)\n", "Requirement already satisfied: nbconvert>=6.4.4 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (7.15.0)\n", "Requirement already satisfied: overrides in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (7.7.0)\n", "Requirement already satisfied: prometheus-client in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.19.0)\n", "Requirement already satisfied: pyzmq>=24 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (25.1.2)\n", "Requirement already satisfied: send2trash>=1.8.2 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (1.8.2)\n", "Requirement already satisfied: terminado>=0.8.3 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.18.0)\n", "Requirement already satisfied: websocket-client in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (1.7.0)\n", "Requirement already satisfied: platformdirs>=2.5 in /usr/local/lib/python3.11/site-packages (from jupyter-core->jupyterlab) (4.2.0)\n", "Requirement already satisfied: babel>=2.10 in /usr/local/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab) (2.14.0)\n", "Requirement already satisfied: json5>=0.9.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab) (0.9.14)\n", "Requirement already satisfied: jsonschema>=4.18.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab) (4.21.1)\n", "Requirement already satisfied: requests>=2.31 in /usr/local/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab) (2.31.0)\n", "Collecting colorama (from nbdime~=4.0.1->jupyterlab-git)\n", " Downloading colorama-0.4.6-py2.py3-none-any.whl.metadata (17 kB)\n", "Collecting gitpython!=2.1.4,!=2.1.5,!=2.1.6 (from nbdime~=4.0.1->jupyterlab-git)\n", " Downloading GitPython-3.1.42-py3-none-any.whl.metadata (12 kB)\n", "Collecting jupyter-server-mathjax>=0.2.2 (from nbdime~=4.0.1->jupyterlab-git)\n", " Downloading jupyter_server_mathjax-0.2.6-py3-none-any.whl (3.1 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m332.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: pygments in /usr/local/lib/python3.11/site-packages (from nbdime~=4.0.1->jupyterlab-git) (2.17.2)\n", "Requirement already satisfied: fastjsonschema in /usr/local/lib/python3.11/site-packages (from nbformat->jupyterlab-git) (2.19.1)\n", "Requirement already satisfied: comm>=0.1.1 in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (0.2.1)\n", "Requirement already satisfied: debugpy>=1.6.5 in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (1.8.0)\n", "Requirement already satisfied: ipython>=7.23.1 in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (8.21.0)\n", "Requirement already satisfied: matplotlib-inline>=0.1 in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (0.1.6)\n", "Requirement already satisfied: nest-asyncio in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (1.6.0)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (5.9.8)\n", "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.11/site-packages (from pexpect->jupyterlab-git) (0.7.0)\n", "Collecting gitdb<5,>=4.0.1 (from gitpython!=2.1.4,!=2.1.5,!=2.1.6->nbdime~=4.0.1->jupyterlab-git)\n", " Downloading gitdb-4.0.11-py3-none-any.whl.metadata (1.2 kB)\n", "Requirement already satisfied: decorator in /usr/local/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel->jupyterlab) (5.1.1)\n", "Requirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel->jupyterlab) (0.19.1)\n", "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in /usr/local/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel->jupyterlab) (3.0.43)\n", "Requirement already satisfied: stack-data in /usr/local/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel->jupyterlab) (0.6.3)\n", "Requirement already satisfied: attrs>=22.2.0 in /usr/local/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.19.0->jupyterlab) (23.2.0)\n", "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.19.0->jupyterlab) (2023.12.1)\n", "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.19.0->jupyterlab) (0.33.0)\n", "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.19.0->jupyterlab) (0.17.1)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/site-packages (from jupyter-client>=7.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (2.8.2)\n", "Requirement already satisfied: python-json-logger>=2.0.4 in /usr/local/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (2.0.7)\n", "Requirement already satisfied: pyyaml>=5.3 in /usr/local/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (6.0.1)\n", "Requirement already satisfied: rfc3339-validator in /usr/local/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (0.1.4)\n", "Requirement already satisfied: rfc3986-validator>=0.1.1 in /usr/local/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (0.1.1)\n", "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (4.12.3)\n", "Requirement already satisfied: bleach!=5.0.0 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (6.1.0)\n", "Requirement already satisfied: defusedxml in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.7.1)\n", "Requirement already satisfied: jupyterlab-pygments in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.3.0)\n", "Requirement already satisfied: mistune<4,>=2.0.3 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (3.0.2)\n", "Requirement already satisfied: nbclient>=0.5.0 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.9.0)\n", "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (1.5.1)\n", "Requirement already satisfied: tinycss2 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (1.2.1)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/site-packages (from requests>=2.31->jupyterlab-server<3,>=2.19.0->jupyterlab) (2.1.1)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/site-packages (from requests>=2.31->jupyterlab-server<3,>=2.19.0->jupyterlab) (2.2.0)\n", "Requirement already satisfied: argon2-cffi-bindings in /usr/local/lib/python3.11/site-packages (from argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab) (21.2.0)\n", "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.11/site-packages (from bleach!=5.0.0->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (1.16.0)\n", "Requirement already satisfied: webencodings in /usr/local/lib/python3.11/site-packages (from bleach!=5.0.0->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.5.1)\n", "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->gitpython!=2.1.4,!=2.1.5,!=2.1.6->nbdime~=4.0.1->jupyterlab-git)\n", " Downloading smmap-5.0.1-py3-none-any.whl.metadata (4.3 kB)\n", "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /usr/local/lib/python3.11/site-packages (from jedi>=0.16->ipython>=7.23.1->ipykernel->jupyterlab) (0.8.3)\n", "Requirement already satisfied: fqdn in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.5.1)\n", "Requirement already satisfied: isoduration in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (20.11.0)\n", "Requirement already satisfied: jsonpointer>1.13 in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (2.4)\n", "Requirement already satisfied: uri-template in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.3.0)\n", "Requirement already satisfied: webcolors>=1.11 in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.13)\n", "Requirement already satisfied: wcwidth in /usr/local/lib/python3.11/site-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython>=7.23.1->ipykernel->jupyterlab) (0.2.13)\n", "Requirement already satisfied: cffi>=1.0.1 in /usr/local/lib/python3.11/site-packages (from argon2-cffi-bindings->argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab) (1.16.0)\n", "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.11/site-packages (from beautifulsoup4->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (2.5)\n", "Requirement already satisfied: executing>=1.2.0 in /usr/local/lib/python3.11/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyterlab) (2.0.1)\n", "Requirement already satisfied: asttokens>=2.1.0 in /usr/local/lib/python3.11/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyterlab) (2.4.1)\n", "Requirement already satisfied: pure-eval in /usr/local/lib/python3.11/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyterlab) (0.2.2)\n", "Requirement already satisfied: pycparser in /usr/local/lib/python3.11/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab) (2.21)\n", "Requirement already satisfied: arrow>=0.15.0 in /usr/local/lib/python3.11/site-packages (from isoduration->jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.3.0)\n", "Requirement already satisfied: types-python-dateutil>=2.8.10 in /usr/local/lib/python3.11/site-packages (from arrow>=0.15.0->isoduration->jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (2.8.19.20240106)\n", "Downloading jupyterlab-4.1.2-py3-none-any.whl (11.4 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.4/11.4 MB\u001b[0m \u001b[31m271.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading jupyterlab_git-0.50.0-py3-none-any.whl (1.2 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m550.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading nbdime-4.0.1-py3-none-any.whl (5.9 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.9/5.9 MB\u001b[0m \u001b[31m254.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading GitPython-3.1.42-py3-none-any.whl (195 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m195.4/195.4 kB\u001b[0m \u001b[31m437.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", "Downloading gitdb-4.0.11-py3-none-any.whl (62 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m299.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading smmap-5.0.1-py3-none-any.whl (24 kB)\n", "Installing collected packages: smmap, colorama, gitdb, gitpython, jupyter-server-mathjax, nbdime, jupyterlab, jupyterlab-git\n", " Attempting uninstall: jupyterlab\n", " Found existing installation: jupyterlab 4.1.0\n", " Uninstalling jupyterlab-4.1.0:\n", " Successfully uninstalled jupyterlab-4.1.0\n", "Successfully installed colorama-0.4.6 gitdb-4.0.11 gitpython-3.1.42 jupyter-server-mathjax-0.2.6 jupyterlab-4.1.2 jupyterlab-git-0.50.0 nbdime-4.0.1 smmap-5.0.1\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ], "source": [ "pip install --upgrade jupyterlab jupyterlab-git" ] }, { "cell_type": "code", "execution_count": 14, "id": "05d58512-a9e2-4319-88bf-9331c6a0584c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", " _| _| _| _| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _|_|_|_| _|_| _|_|_| _|_|_|_|\n", " _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _|\n", " _|_|_|_| _| _| _| _|_| _| _|_| _| _| _| _| _| _|_| _|_|_| _|_|_|_| _| _|_|_|\n", " _| _| _| _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _|\n", " _| _| _|_| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _| _| _| _|_|_| _|_|_|_|\n", "\n", " To login, `huggingface_hub` requires a token generated from https://huggingface.co/settings/tokens .\n" ] }, { "name": "stdin", "output_type": "stream", "text": [ "Token: ········\n", "Add token as git credential? (Y/n) n\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Token is valid (permission: write).\n", "Your token has been saved to /root/.cache/huggingface/token\n", "Login successful\n" ] } ], "source": [ "from huggingface_hub import interpreter_login\n", "interpreter_login()" ] }, { "cell_type": "code", "execution_count": 15, "id": "bcb01d1b-9a48-46fe-b020-51a4d61df532", "metadata": { "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Collecting torch==2.1.0\n", " Downloading torch-2.1.0-cp311-cp311-manylinux1_x86_64.whl.metadata (25 kB)\n", "Collecting pytorch-lightning==1.9.4\n", " Downloading pytorch_lightning-1.9.4-py3-none-any.whl.metadata (22 kB)\n", "Collecting accelerate==0.21.0\n", " Downloading accelerate-0.21.0-py3-none-any.whl.metadata (17 kB)\n", "Collecting tokenizers==0.13.3\n", " Downloading tokenizers-0.13.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n", "Requirement already satisfied: transformers in /usr/local/lib/python3.11/site-packages (4.37.2)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (3.13.1)\n", "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (4.9.0)\n", "Requirement already satisfied: sympy in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (1.12)\n", "Requirement already satisfied: networkx in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (3.2.1)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (3.1.3)\n", "Requirement already satisfied: fsspec in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (2023.10.0)\n", "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.105)\n", "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.105)\n", "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.105)\n", "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (8.9.2.26)\n", "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.3.1)\n", "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (11.0.2.54)\n", "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (10.3.2.106)\n", "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (11.4.5.107)\n", "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.0.106)\n", "Collecting nvidia-nccl-cu12==2.18.1 (from torch==2.1.0)\n", " Downloading nvidia_nccl_cu12-2.18.1-py3-none-manylinux1_x86_64.whl.metadata (1.8 kB)\n", "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.105)\n", "Collecting triton==2.1.0 (from torch==2.1.0)\n", " Downloading triton-2.1.0-0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.3 kB)\n", "Requirement already satisfied: numpy>=1.17.2 in /usr/local/lib/python3.11/site-packages (from pytorch-lightning==1.9.4) (1.26.4)\n", "Requirement already satisfied: tqdm>=4.57.0 in /usr/local/lib/python3.11/site-packages (from pytorch-lightning==1.9.4) (4.66.2)\n", "Requirement already satisfied: PyYAML>=5.4 in /usr/local/lib/python3.11/site-packages (from pytorch-lightning==1.9.4) (6.0.1)\n", "Collecting torchmetrics>=0.7.0 (from pytorch-lightning==1.9.4)\n", " Downloading torchmetrics-1.3.1-py3-none-any.whl.metadata (19 kB)\n", "Requirement already satisfied: packaging>=17.1 in /usr/local/lib/python3.11/site-packages (from pytorch-lightning==1.9.4) (23.2)\n", "Collecting lightning-utilities>=0.6.0.post0 (from pytorch-lightning==1.9.4)\n", " Downloading lightning_utilities-0.10.1-py3-none-any.whl.metadata (4.8 kB)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.11/site-packages (from accelerate==0.21.0) (5.9.8)\n", "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.11/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch==2.1.0) (12.3.101)\n", "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /usr/local/lib/python3.11/site-packages (from transformers) (0.20.3)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.11/site-packages (from transformers) (2023.12.25)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.11/site-packages (from transformers) (2.31.0)\n", "INFO: pip is looking at multiple versions of transformers to determine which version is compatible with other requirements. This could take a while.\n", "Collecting transformers\n", " Downloading transformers-4.37.1-py3-none-any.whl.metadata (129 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.4/129.4 kB\u001b[0m \u001b[31m447.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.37.0-py3-none-any.whl.metadata (129 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.4/129.4 kB\u001b[0m \u001b[31m524.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.36.2-py3-none-any.whl.metadata (126 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.8/126.8 kB\u001b[0m \u001b[31m120.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.36.1-py3-none-any.whl.metadata (126 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.8/126.8 kB\u001b[0m \u001b[31m212.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.36.0-py3-none-any.whl.metadata (126 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.8/126.8 kB\u001b[0m \u001b[31m397.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.35.2-py3-none-any.whl.metadata (123 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m123.5/123.5 kB\u001b[0m \u001b[31m298.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.35.1-py3-none-any.whl.metadata (123 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m123.1/123.1 kB\u001b[0m \u001b[31m433.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hINFO: pip is still looking at multiple versions of transformers to determine which version is compatible with other requirements. This could take a while.\n", " Downloading transformers-4.35.0-py3-none-any.whl.metadata (123 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m123.1/123.1 kB\u001b[0m \u001b[31m433.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.34.1-py3-none-any.whl.metadata (121 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.5/121.5 kB\u001b[0m \u001b[31m205.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.34.0-py3-none-any.whl.metadata (121 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.5/121.5 kB\u001b[0m \u001b[31m529.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.33.3-py3-none-any.whl.metadata (119 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.9/119.9 kB\u001b[0m \u001b[31m221.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.11/site-packages (from transformers) (0.4.2)\n", "Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /usr/local/lib/python3.11/site-packages (from fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (3.8.3)\n", "Requirement already satisfied: setuptools in /usr/local/lib/python3.11/site-packages (from lightning-utilities>=0.6.0.post0->pytorch-lightning==1.9.4) (68.1.2)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/site-packages (from jinja2->torch==2.1.0) (2.1.5)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2.1.1)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2.2.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2024.2.2)\n", "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.11/site-packages (from sympy->torch==2.1.0) (1.3.0)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (23.2.0)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (6.0.5)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (4.0.3)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (1.9.4)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (1.4.1)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (1.3.1)\n", "Downloading torch-2.1.0-cp311-cp311-manylinux1_x86_64.whl (670.2 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m670.2/670.2 MB\u001b[0m \u001b[31m179.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading pytorch_lightning-1.9.4-py3-none-any.whl (827 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m827.8/827.8 kB\u001b[0m \u001b[31m456.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading accelerate-0.21.0-py3-none-any.whl (244 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m244.2/244.2 kB\u001b[0m \u001b[31m395.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading tokenizers-0.13.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m311.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading nvidia_nccl_cu12-2.18.1-py3-none-manylinux1_x86_64.whl (209.8 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m209.8/209.8 MB\u001b[0m \u001b[31m315.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading triton-2.1.0-0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (89.2 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m89.2/89.2 MB\u001b[0m \u001b[31m277.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading transformers-4.33.3-py3-none-any.whl (7.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m342.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading lightning_utilities-0.10.1-py3-none-any.whl (24 kB)\n", "Downloading torchmetrics-1.3.1-py3-none-any.whl (840 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m840.4/840.4 kB\u001b[0m \u001b[31m591.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hInstalling collected packages: tokenizers, triton, nvidia-nccl-cu12, lightning-utilities, transformers, torch, torchmetrics, accelerate, pytorch-lightning\n", " Attempting uninstall: tokenizers\n", " Found existing installation: tokenizers 0.15.2\n", " Uninstalling tokenizers-0.15.2:\n", " Successfully uninstalled tokenizers-0.15.2\n", " Attempting uninstall: triton\n", " Found existing installation: triton 2.2.0\n", " Uninstalling triton-2.2.0:\n", " Successfully uninstalled triton-2.2.0\n", " Attempting uninstall: nvidia-nccl-cu12\n", " Found existing installation: nvidia-nccl-cu12 2.19.3\n", " Uninstalling nvidia-nccl-cu12-2.19.3:\n", " Successfully uninstalled nvidia-nccl-cu12-2.19.3\n", " Attempting uninstall: transformers\n", " Found existing installation: transformers 4.37.2\n", " Uninstalling transformers-4.37.2:\n", " Successfully uninstalled transformers-4.37.2\n", " Attempting uninstall: torch\n", " Found existing installation: torch 2.2.0\n", " Uninstalling torch-2.2.0:\n", " Successfully uninstalled torch-2.2.0\n", " Attempting uninstall: accelerate\n", " Found existing installation: accelerate 0.27.2\n", " Uninstalling accelerate-0.27.2:\n", " Successfully uninstalled accelerate-0.27.2\n", "Successfully installed accelerate-0.21.0 lightning-utilities-0.10.1 nvidia-nccl-cu12-2.18.1 pytorch-lightning-1.9.4 tokenizers-0.13.3 torch-2.1.0 torchmetrics-1.3.1 transformers-4.33.3 triton-2.1.0\n" ] } ], "source": [ " !pip install torch==2.1.0 pytorch-lightning==1.9.4 accelerate==0.21.0 tokenizers==0.13.3 transformers" ] }, { "cell_type": "code", "execution_count": 16, "id": "d76b4865-df75-4793-b8b9-d97523445945", "metadata": {}, "outputs": [], "source": [ "model.enable_input_require_grads()" ] }, { "cell_type": "code", "execution_count": null, "id": "3bf553b6-b26c-49c3-9407-74c8d53a395e", "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/html": [ "\n", "
Step | \n", "Training Loss | \n", "
---|---|
50 | \n", "1.458400 | \n", "
100 | \n", "1.300500 | \n", "
150 | \n", "1.163600 | \n", "
200 | \n", "1.231700 | \n", "
250 | \n", "1.215000 | \n", "
300 | \n", "1.167400 | \n", "
350 | \n", "1.061700 | \n", "
400 | \n", "1.182300 | \n", "
450 | \n", "1.107400 | \n", "
500 | \n", "1.168000 | \n", "
550 | \n", "1.102500 | \n", "
600 | \n", "1.121800 | \n", "
650 | \n", "1.145500 | \n", "
700 | \n", "1.145000 | \n", "
750 | \n", "1.155200 | \n", "
800 | \n", "1.099600 | \n", "
850 | \n", "1.090800 | \n", "
900 | \n", "1.080600 | \n", "
950 | \n", "1.129900 | \n", "
1000 | \n", "1.061500 | \n", "
1050 | \n", "1.099200 | \n", "
1100 | \n", "1.122900 | \n", "
1150 | \n", "1.128500 | \n", "
1200 | \n", "1.085700 | \n", "
1250 | \n", "1.165000 | \n", "
1300 | \n", "1.083300 | \n", "
1350 | \n", "1.082900 | \n", "
1400 | \n", "1.131900 | \n", "
1450 | \n", "1.066800 | \n", "
1500 | \n", "1.128600 | \n", "
1550 | \n", "1.159600 | \n", "
1600 | \n", "1.120100 | \n", "
1650 | \n", "1.042100 | \n", "
1700 | \n", "1.105800 | \n", "
1750 | \n", "1.124700 | \n", "
1800 | \n", "1.043800 | \n", "
1850 | \n", "1.120900 | \n", "
1900 | \n", "1.081700 | \n", "
1950 | \n", "1.123200 | \n", "
2000 | \n", "1.074400 | \n", "
2050 | \n", "1.166000 | \n", "
2100 | \n", "1.097300 | \n", "
2150 | \n", "0.989200 | \n", "
2200 | \n", "0.988200 | \n", "
2250 | \n", "1.080400 | \n", "
2300 | \n", "1.100700 | \n", "
2350 | \n", "1.074600 | \n", "
2400 | \n", "1.009900 | \n", "
2450 | \n", "1.117800 | \n", "
2500 | \n", "1.131300 | \n", "
2550 | \n", "1.033500 | \n", "
2600 | \n", "1.094400 | \n", "
2650 | \n", "1.072600 | \n", "
2700 | \n", "1.099600 | \n", "
2750 | \n", "0.988700 | \n", "
2800 | \n", "1.009600 | \n", "
2850 | \n", "1.044400 | \n", "
2900 | \n", "1.075700 | \n", "
2950 | \n", "0.983500 | \n", "
3000 | \n", "0.982800 | \n", "
3050 | \n", "1.104500 | \n", "
3100 | \n", "1.155100 | \n", "
3150 | \n", "1.029100 | \n", "
3200 | \n", "1.047900 | \n", "
3250 | \n", "1.045000 | \n", "
3300 | \n", "0.987200 | \n", "
3350 | \n", "0.967300 | \n", "
3400 | \n", "1.063000 | \n", "
3450 | \n", "1.003900 | \n", "
3500 | \n", "0.997600 | \n", "
3550 | \n", "1.066800 | \n", "
3600 | \n", "1.099700 | \n", "
3650 | \n", "1.045500 | \n", "
3700 | \n", "1.060900 | \n", "
3750 | \n", "1.084000 | \n", "
3800 | \n", "1.041400 | \n", "
3850 | \n", "1.074100 | \n", "
3900 | \n", "0.991600 | \n", "
3950 | \n", "1.074900 | \n", "
4000 | \n", "1.082100 | \n", "
4050 | \n", "1.051500 | \n", "
4100 | \n", "1.052600 | \n", "
4150 | \n", "1.029600 | \n", "
4200 | \n", "0.997700 | \n", "
4250 | \n", "0.982800 | \n", "
4300 | \n", "0.984900 | \n", "
4350 | \n", "1.020700 | \n", "
4400 | \n", "1.021000 | \n", "
4450 | \n", "0.967500 | \n", "
4500 | \n", "1.000800 | \n", "
4550 | \n", "0.973400 | \n", "
4600 | \n", "0.986600 | \n", "
4650 | \n", "0.985700 | \n", "
4700 | \n", "0.924400 | \n", "
4750 | \n", "0.978400 | \n", "
4800 | \n", "1.031400 | \n", "
4850 | \n", "0.952300 | \n", "
4900 | \n", "0.995800 | \n", "
4950 | \n", "0.985900 | \n", "
5000 | \n", "0.981700 | \n", "
5050 | \n", "0.957800 | \n", "
5100 | \n", "0.908500 | \n", "
5150 | \n", "0.957400 | \n", "
5200 | \n", "1.009800 | \n", "
5250 | \n", "0.973400 | \n", "
5300 | \n", "1.014600 | \n", "
5350 | \n", "0.925700 | \n", "
5400 | \n", "1.018200 | \n", "
5450 | \n", "1.001800 | \n", "
5500 | \n", "1.015900 | \n", "
5550 | \n", "0.976700 | \n", "
5600 | \n", "0.994300 | \n", "
5650 | \n", "0.986000 | \n", "
5700 | \n", "0.934200 | \n", "
5750 | \n", "0.955100 | \n", "
5800 | \n", "0.946400 | \n", "
5850 | \n", "0.976700 | \n", "
5900 | \n", "0.983900 | \n", "
5950 | \n", "0.961400 | \n", "
6000 | \n", "0.965800 | \n", "
6050 | \n", "0.932600 | \n", "
6100 | \n", "0.978700 | \n", "
6150 | \n", "0.968300 | \n", "
6200 | \n", "1.000700 | \n", "
6250 | \n", "0.989300 | \n", "
6300 | \n", "0.964000 | \n", "
6350 | \n", "0.959000 | \n", "
6400 | \n", "0.918600 | \n", "
6450 | \n", "0.965600 | \n", "
6500 | \n", "0.937600 | \n", "
6550 | \n", "1.037600 | \n", "
6600 | \n", "0.991400 | \n", "
6650 | \n", "1.000300 | \n", "
6700 | \n", "1.007200 | \n", "
6750 | \n", "1.005800 | \n", "
6800 | \n", "1.071100 | \n", "
6850 | \n", "0.990100 | \n", "
6900 | \n", "0.972700 | \n", "
6950 | \n", "0.923800 | \n", "
7000 | \n", "0.968300 | \n", "
7050 | \n", "0.895400 | \n", "
7100 | \n", "0.965400 | \n", "
7150 | \n", "0.920500 | \n", "
7200 | \n", "0.950700 | \n", "
7250 | \n", "0.961400 | \n", "
7300 | \n", "0.926500 | \n", "
7350 | \n", "0.932500 | \n", "
7400 | \n", "0.918000 | \n", "
7450 | \n", "0.959700 | \n", "
7500 | \n", "0.970300 | \n", "
7550 | \n", "0.920800 | \n", "
7600 | \n", "0.944500 | \n", "
7650 | \n", "1.014300 | \n", "
7700 | \n", "0.960400 | \n", "
7750 | \n", "0.953300 | \n", "
7800 | \n", "0.946200 | \n", "
7850 | \n", "0.965100 | \n", "
7900 | \n", "0.942200 | \n", "
7950 | \n", "0.952900 | \n", "
8000 | \n", "0.895400 | \n", "
8050 | \n", "0.974000 | \n", "
8100 | \n", "0.990500 | \n", "
8150 | \n", "0.972100 | \n", "
8200 | \n", "0.968300 | \n", "
8250 | \n", "0.999500 | \n", "
8300 | \n", "0.924000 | \n", "
8350 | \n", "0.966000 | \n", "
8400 | \n", "0.906600 | \n", "
8450 | \n", "0.955800 | \n", "
8500 | \n", "0.895400 | \n", "
8550 | \n", "0.878600 | \n", "
8600 | \n", "0.909200 | \n", "
8650 | \n", "0.952800 | \n", "
8700 | \n", "0.876500 | \n", "
8750 | \n", "0.932500 | \n", "
8800 | \n", "0.892700 | \n", "
8850 | \n", "0.956300 | \n", "
8900 | \n", "0.948300 | \n", "
8950 | \n", "0.939900 | \n", "
9000 | \n", "0.893700 | \n", "
9050 | \n", "0.947700 | \n", "
9100 | \n", "0.922700 | \n", "
9150 | \n", "0.865500 | \n", "
9200 | \n", "0.883100 | \n", "
9250 | \n", "0.863200 | \n", "
9300 | \n", "0.901800 | \n", "
9350 | \n", "0.914500 | \n", "
9400 | \n", "0.922100 | \n", "
9450 | \n", "0.897600 | \n", "
9500 | \n", "0.911700 | \n", "
9550 | \n", "0.903800 | \n", "
9600 | \n", "0.965100 | \n", "
9650 | \n", "0.851700 | \n", "
9700 | \n", "0.935000 | \n", "
9750 | \n", "0.954300 | \n", "
9800 | \n", "0.908300 | \n", "
9850 | \n", "0.892500 | \n", "
9900 | \n", "0.936300 | \n", "
9950 | \n", "0.986800 | \n", "
10000 | \n", "0.915300 | \n", "
"
],
"text/plain": [
"