{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "b3b92bc7-d105-405f-970d-804d298b9976", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting accelerate\n", " Downloading accelerate-0.27.2-py3-none-any.whl.metadata (18 kB)\n", "Collecting transformers\n", " Downloading transformers-4.37.2-py3-none-any.whl.metadata (129 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.4/129.4 kB\u001b[0m \u001b[31m80.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting einops\n", " Downloading einops-0.7.0-py3-none-any.whl.metadata (13 kB)\n", "Collecting datasets\n", " Downloading datasets-2.17.0-py3-none-any.whl.metadata (20 kB)\n", "Collecting peft\n", " Downloading peft-0.8.2-py3-none-any.whl.metadata (25 kB)\n", "Collecting bitsandbytes\n", " Downloading bitsandbytes-0.42.0-py3-none-any.whl.metadata (9.9 kB)\n", "Collecting torch\n", " Downloading torch-2.2.0-cp311-cp311-manylinux1_x86_64.whl.metadata (25 kB)\n", "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.11/site-packages (from accelerate) (1.26.4)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/site-packages (from accelerate) (23.2)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.11/site-packages (from accelerate) (5.9.8)\n", "Requirement already satisfied: pyyaml in /usr/local/lib/python3.11/site-packages (from accelerate) (6.0.1)\n", "Collecting huggingface-hub (from accelerate)\n", " Downloading huggingface_hub-0.20.3-py3-none-any.whl.metadata (12 kB)\n", "Collecting safetensors>=0.3.1 (from accelerate)\n", " Downloading safetensors-0.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.8 kB)\n", "Collecting filelock (from transformers)\n", " Downloading filelock-3.13.1-py3-none-any.whl.metadata (2.8 kB)\n", "Collecting regex!=2019.12.17 (from transformers)\n", " Downloading regex-2023.12.25-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (40 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.9/40.9 kB\u001b[0m \u001b[31m416.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.11/site-packages (from transformers) (2.31.0)\n", "Collecting tokenizers<0.19,>=0.14 (from transformers)\n", " Downloading tokenizers-0.15.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n", "Collecting tqdm>=4.27 (from transformers)\n", " Downloading tqdm-4.66.2-py3-none-any.whl.metadata (57 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.6/57.6 kB\u001b[0m \u001b[31m156.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting pyarrow>=12.0.0 (from datasets)\n", " Downloading pyarrow-15.0.0-cp311-cp311-manylinux_2_28_x86_64.whl.metadata (3.0 kB)\n", "Collecting pyarrow-hotfix (from datasets)\n", " Downloading pyarrow_hotfix-0.6-py3-none-any.whl.metadata (3.6 kB)\n", "Collecting dill<0.3.9,>=0.3.0 (from datasets)\n", " Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n", "Collecting pandas (from datasets)\n", " Downloading pandas-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (19 kB)\n", "Collecting xxhash (from datasets)\n", " Downloading xxhash-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n", "Collecting multiprocess (from datasets)\n", " Downloading multiprocess-0.70.16-py311-none-any.whl.metadata (7.2 kB)\n", "Collecting fsspec<=2023.10.0,>=2023.1.0 (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets)\n", " Downloading fsspec-2023.10.0-py3-none-any.whl.metadata (6.8 kB)\n", "Requirement already satisfied: aiohttp in /usr/local/lib/python3.11/site-packages (from datasets) (3.8.3)\n", "Collecting scipy (from bitsandbytes)\n", " Downloading scipy-1.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (60 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.4/60.4 kB\u001b[0m \u001b[31m432.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.11/site-packages (from torch) (4.9.0)\n", "Collecting sympy (from torch)\n", " Downloading sympy-1.12-py3-none-any.whl.metadata (12 kB)\n", "Collecting networkx (from torch)\n", " Downloading networkx-3.2.1-py3-none-any.whl.metadata (5.2 kB)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/site-packages (from torch) (3.1.3)\n", "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch)\n", " Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch)\n", " Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch)\n", " Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n", "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch)\n", " Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n", "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch)\n", " Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch)\n", " Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Collecting nvidia-curand-cu12==10.3.2.106 (from torch)\n", " Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch)\n", " Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n", "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch)\n", " Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n", "Collecting nvidia-nccl-cu12==2.19.3 (from torch)\n", " Downloading nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl.metadata (1.8 kB)\n", "Collecting nvidia-nvtx-cu12==12.1.105 (from torch)\n", " Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.7 kB)\n", "Collecting triton==2.2.0 (from torch)\n", " Downloading triton-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.4 kB)\n", "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch)\n", " Downloading nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (23.2.0)\n", "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (2.1.1)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (6.0.5)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (4.0.3)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (1.9.4)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (1.4.1)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (1.3.1)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2.2.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2024.2.2)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/site-packages (from jinja2->torch) (2.1.5)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/site-packages (from pandas->datasets) (2.8.2)\n", "Collecting pytz>=2020.1 (from pandas->datasets)\n", " Downloading pytz-2024.1-py2.py3-none-any.whl.metadata (22 kB)\n", "Collecting tzdata>=2022.7 (from pandas->datasets)\n", " Downloading tzdata-2024.1-py2.py3-none-any.whl.metadata (1.4 kB)\n", "Collecting mpmath>=0.19 (from sympy->torch)\n", " Downloading mpmath-1.3.0-py3-none-any.whl (536 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m536.2/536.2 kB\u001b[0m \u001b[31m344.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: six>=1.5 in /usr/local/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n", "Downloading accelerate-0.27.2-py3-none-any.whl (279 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m280.0/280.0 kB\u001b[0m \u001b[31m352.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading transformers-4.37.2-py3-none-any.whl (8.4 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.4/8.4 MB\u001b[0m \u001b[31m235.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading einops-0.7.0-py3-none-any.whl (44 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.6/44.6 kB\u001b[0m \u001b[31m358.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading datasets-2.17.0-py3-none-any.whl (536 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m536.6/536.6 kB\u001b[0m \u001b[31m461.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading peft-0.8.2-py3-none-any.whl (183 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m183.4/183.4 kB\u001b[0m \u001b[31m386.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading bitsandbytes-0.42.0-py3-none-any.whl (105.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m105.0/105.0 MB\u001b[0m \u001b[31m164.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading torch-2.2.0-cp311-cp311-manylinux1_x86_64.whl (755.5 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m755.5/755.5 MB\u001b[0m \u001b[31m257.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m272.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m191.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m262.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m553.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m319.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m244.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m294.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m276.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m163.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m166.0/166.0 MB\u001b[0m \u001b[31m207.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m338.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading triton-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (167.9 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m167.9/167.9 MB\u001b[0m \u001b[31m217.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading dill-0.3.8-py3-none-any.whl (116 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m340.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading fsspec-2023.10.0-py3-none-any.whl (166 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m166.4/166.4 kB\u001b[0m \u001b[31m579.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading huggingface_hub-0.20.3-py3-none-any.whl (330 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m330.1/330.1 kB\u001b[0m \u001b[31m454.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading pyarrow-15.0.0-cp311-cp311-manylinux_2_28_x86_64.whl (38.3 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m38.3/38.3 MB\u001b[0m \u001b[31m283.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading regex-2023.12.25-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (785 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m785.1/785.1 kB\u001b[0m \u001b[31m547.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading safetensors-0.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m538.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading tokenizers-0.15.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.6/3.6 MB\u001b[0m \u001b[31m80.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mta \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading tqdm-4.66.2-py3-none-any.whl (78 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.3/78.3 kB\u001b[0m \u001b[31m97.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading filelock-3.13.1-py3-none-any.whl (11 kB)\n", "Downloading multiprocess-0.70.16-py311-none-any.whl (143 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.5/143.5 kB\u001b[0m \u001b[31m396.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading networkx-3.2.1-py3-none-any.whl (1.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m464.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading pandas-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (13.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.0/13.0 MB\u001b[0m \u001b[31m256.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading pyarrow_hotfix-0.6-py3-none-any.whl (7.9 kB)\n", "Downloading scipy-1.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (38.4 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m38.4/38.4 MB\u001b[0m \u001b[31m282.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading sympy-1.12-py3-none-any.whl (5.7 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.7/5.7 MB\u001b[0m \u001b[31m180.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading xxhash-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.8/194.8 kB\u001b[0m \u001b[31m540.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading pytz-2024.1-py2.py3-none-any.whl (505 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m505.5/505.5 kB\u001b[0m \u001b[31m586.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading tzdata-2024.1-py2.py3-none-any.whl (345 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m345.4/345.4 kB\u001b[0m \u001b[31m585.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl (20.5 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m20.5/20.5 MB\u001b[0m \u001b[31m199.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hInstalling collected packages: pytz, mpmath, xxhash, tzdata, tqdm, sympy, scipy, safetensors, regex, pyarrow-hotfix, pyarrow, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, networkx, fsspec, filelock, einops, dill, triton, pandas, nvidia-cusparse-cu12, nvidia-cudnn-cu12, multiprocess, huggingface-hub, bitsandbytes, tokenizers, nvidia-cusolver-cu12, transformers, torch, datasets, accelerate, peft\n", "Successfully installed accelerate-0.27.2 bitsandbytes-0.42.0 datasets-2.17.0 dill-0.3.8 einops-0.7.0 filelock-3.13.1 fsspec-2023.10.0 huggingface-hub-0.20.3 mpmath-1.3.0 multiprocess-0.70.16 networkx-3.2.1 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.3.101 nvidia-nvtx-cu12-12.1.105 pandas-2.2.0 peft-0.8.2 pyarrow-15.0.0 pyarrow-hotfix-0.6 pytz-2024.1 regex-2023.12.25 safetensors-0.4.2 scipy-1.12.0 sympy-1.12 tokenizers-0.15.2 torch-2.2.0 tqdm-4.66.2 transformers-4.37.2 triton-2.2.0 tzdata-2024.1 xxhash-3.4.1\n" ] } ], "source": [ "!pip install accelerate transformers einops datasets peft bitsandbytes torch" ] }, { "cell_type": "code", "execution_count": 2, "id": "46a53303-b585-4b02-956f-4af173410e25", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] } ], "source": [ "import torch\n", "from datasets import load_dataset, Dataset\n", "from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, Trainer, DataCollatorForLanguageModeling, BitsAndBytesConfig\n", "from peft import LoraConfig, get_peft_model\n", "import os" ] }, { "cell_type": "code", "execution_count": 3, "id": "ffa3ee11-3929-4032-8758-322cda3a912e", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "tokenizer_config.json: 100%|██████████| 7.34k/7.34k [00:00<00:00, 22.1MB/s]\n", "vocab.json: 100%|██████████| 798k/798k [00:00<00:00, 3.38MB/s]\n", "merges.txt: 100%|██████████| 456k/456k [00:00<00:00, 50.2MB/s]\n", "tokenizer.json: 100%|██████████| 2.11M/2.11M [00:00<00:00, 45.9MB/s]\n", "added_tokens.json: 100%|██████████| 1.08k/1.08k [00:00<00:00, 8.97MB/s]\n", "special_tokens_map.json: 100%|██████████| 99.0/99.0 [00:00<00:00, 779kB/s]\n", "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] } ], "source": [ "tokenizer = AutoTokenizer.from_pretrained(\"microsoft/phi-2\", trust_remote_code=True)\n", "tokenizer.pad_token = tokenizer.eos_token" ] }, { "cell_type": "code", "execution_count": 4, "id": "b17096e6-c633-4d88-8cfc-fb738c1e4ca0", "metadata": { "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "config.json: 100%|██████████| 863/863 [00:00<00:00, 6.70MB/s]\n", "configuration_phi.py: 100%|██████████| 9.26k/9.26k [00:00<00:00, 46.0MB/s]\n", "A new version of the following files was downloaded from https://huggingface.co/microsoft/phi-2:\n", "- configuration_phi.py\n", ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n", "modeling_phi.py: 100%|██████████| 62.7k/62.7k [00:00<00:00, 18.8MB/s]\n", "A new version of the following files was downloaded from https://huggingface.co/microsoft/phi-2:\n", "- modeling_phi.py\n", ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n", "model.safetensors.index.json: 100%|██████████| 35.7k/35.7k [00:00<00:00, 126MB/s]\n", "Downloading shards: 0%| | 0/2 [00:00 10\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprint_trainable_parameters\u001b[49m()\n", "File \u001b[0;32m/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py:1688\u001b[0m, in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m 1686\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m _parameters[name]\n\u001b[1;32m 1687\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m_buffers\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__dict__\u001b[39m:\n\u001b[0;32m-> 1688\u001b[0m _buffers \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__dict__\u001b[39m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m_buffers\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m 1689\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01min\u001b[39;00m _buffers:\n\u001b[1;32m 1690\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m _buffers[name]\n", "\u001b[0;31mAttributeError\u001b[0m: 'PhiForCausalLM' object has no attribute 'print_trainable_parameters'" ] } ], "source": [ "# config = LoraConfig(\n", "# r=16,\n", "# lora_alpha=16,\n", "# #target_modules=[\"q_proj\",\"k_proj\",\"v_proj\"],\n", "# lora_dropout=0.05,\n", "# bias=\"none\",\n", "# task_type=\"CAUSAL_LM\"\n", "# )\n", "\n", "# model = get_peft_model(model, config)\n", "# model.print_trainable_parameters()" ] }, { "cell_type": "code", "execution_count": 38, "id": "baeee903-3dce-48b2-93c3-7a697d8c6daf", "metadata": {}, "outputs": [], "source": [ "def tokenize(sample):\n", " model_inps = tokenizer(sample[\"text\"], padding=True, truncation=True, max_length=512)\n", " return model_inps" ] }, { "cell_type": "code", "execution_count": 28, "id": "28a9b24a-a822-4fcb-96b3-d77b7ea30a5f", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.11/site-packages (1.4.0)\n", "Requirement already satisfied: numpy<2.0,>=1.19.5 in /usr/local/lib/python3.11/site-packages (from scikit-learn) (1.26.4)\n", "Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.11/site-packages (from scikit-learn) (1.12.0)\n", "Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.11/site-packages (from scikit-learn) (1.3.2)\n", "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.11/site-packages (from scikit-learn) (3.3.0)\n" ] } ], "source": [ "!pip install scikit-learn" ] }, { "cell_type": "code", "execution_count": 29, "id": "1ee7fd2a-38e4-4f23-a978-0bdeeda64d8b", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "\n", "from sklearn.model_selection import train_test_split\n", "dataset_name='data.csv'\n", "df = pd.read_csv(dataset_name)\n", "#train, test = train_test_split(df, test_size=0.2)" ] }, { "cell_type": "code", "execution_count": 30, "id": "e84c29e2-843e-42c2-8c0f-324d392e671c", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Tokenizing data: 100%|██████████| 16412/16412 [00:02<00:00, 7265.30 examples/s]\n" ] }, { "data": { "text/plain": [ "Dataset({\n", " features: ['input_ids', 'attention_mask'],\n", " num_rows: 16412\n", "})" ] }, "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data_df = df\n", "data_df[\"text\"] = data_df[[\"user\", \"assistant\"]].apply(lambda x: \"question: \" + str(x[\"user\"]) + \" answer: \" + str(x[\"assistant\"]), axis=1)\n", "data = Dataset.from_pandas(data_df)\n", "tokenized_data = data.map(tokenize, batched=True, desc=\"Tokenizing data\", remove_columns=data.column_names)\n", "tokenized_data" ] }, { "cell_type": "code", "execution_count": 44, "id": "ac968254-5338-49df-950d-222b82647407", "metadata": {}, "outputs": [], "source": [ "training_arguments = TrainingArguments(\n", " output_dir=\".\",\n", " per_device_train_batch_size=4,\n", " gradient_accumulation_steps=1,\n", " learning_rate=2e-4,\n", " lr_scheduler_type=\"cosine\",\n", " save_strategy=\"epoch\",\n", " logging_steps=100,\n", " max_steps=200,\n", " num_train_epochs=3,\n", " push_to_hub=True\n", " )" ] }, { "cell_type": "code", "execution_count": 32, "id": "729df2d0-0890-4ac4-adf3-c167a6e9669d", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: huggingface_hub in /usr/local/lib/python3.11/site-packages (0.20.3)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (3.13.1)\n", "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (2023.10.0)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (2.31.0)\n", "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (4.66.2)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (6.0.1)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (4.9.0)\n", "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (23.2)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/site-packages (from requests->huggingface_hub) (2.1.1)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/site-packages (from requests->huggingface_hub) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/site-packages (from requests->huggingface_hub) (2.2.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/site-packages (from requests->huggingface_hub) (2024.2.2)\n" ] } ], "source": [ "!pip install huggingface_hub" ] }, { "cell_type": "code", "execution_count": 33, "id": "525f81a4-eb92-466e-bb9c-cd63122231ab", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: jupyterlab in /usr/local/lib/python3.11/site-packages (4.1.1)\n", "Requirement already satisfied: jupyterlab-git in /usr/local/lib/python3.11/site-packages (0.50.0)\n", "Requirement already satisfied: async-lru>=1.0.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (2.0.4)\n", "Requirement already satisfied: httpx>=0.25.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (0.26.0)\n", "Requirement already satisfied: ipykernel in /usr/local/lib/python3.11/site-packages (from jupyterlab) (6.29.1)\n", "Requirement already satisfied: jinja2>=3.0.3 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (3.1.3)\n", "Requirement already satisfied: jupyter-core in /usr/local/lib/python3.11/site-packages (from jupyterlab) (5.7.1)\n", "Requirement already satisfied: jupyter-lsp>=2.0.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (2.2.2)\n", "Requirement already satisfied: jupyter-server<3,>=2.4.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (2.12.5)\n", "Requirement already satisfied: jupyterlab-server<3,>=2.19.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (2.25.2)\n", "Requirement already satisfied: notebook-shim>=0.2 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (0.2.3)\n", "Requirement already satisfied: packaging in /usr/local/lib/python3.11/site-packages (from jupyterlab) (23.2)\n", "Requirement already satisfied: tornado>=6.2.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (6.4)\n", "Requirement already satisfied: traitlets in /usr/local/lib/python3.11/site-packages (from jupyterlab) (5.14.1)\n", "Requirement already satisfied: nbdime~=4.0.1 in /usr/local/lib/python3.11/site-packages (from jupyterlab-git) (4.0.1)\n", "Requirement already satisfied: nbformat in /usr/local/lib/python3.11/site-packages (from jupyterlab-git) (5.9.2)\n", "Requirement already satisfied: pexpect in /usr/local/lib/python3.11/site-packages (from jupyterlab-git) (4.9.0)\n", "Requirement already satisfied: anyio in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (4.2.0)\n", "Requirement already satisfied: certifi in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (2024.2.2)\n", "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (1.0.2)\n", "Requirement already satisfied: idna in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (3.6)\n", "Requirement already satisfied: sniffio in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (1.3.0)\n", "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.11/site-packages (from httpcore==1.*->httpx>=0.25.0->jupyterlab) (0.14.0)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/site-packages (from jinja2>=3.0.3->jupyterlab) (2.1.5)\n", "Requirement already satisfied: argon2-cffi in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (23.1.0)\n", "Requirement already satisfied: jupyter-client>=7.4.4 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (8.6.0)\n", "Requirement already satisfied: jupyter-events>=0.9.0 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.9.0)\n", "Requirement already satisfied: jupyter-server-terminals in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.5.2)\n", "Requirement already satisfied: nbconvert>=6.4.4 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (7.15.0)\n", "Requirement already satisfied: overrides in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (7.7.0)\n", "Requirement already satisfied: prometheus-client in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.19.0)\n", "Requirement already satisfied: pyzmq>=24 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (25.1.2)\n", "Requirement already satisfied: send2trash>=1.8.2 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (1.8.2)\n", "Requirement already satisfied: terminado>=0.8.3 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.18.0)\n", "Requirement already satisfied: websocket-client in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (1.7.0)\n", "Requirement already satisfied: platformdirs>=2.5 in /usr/local/lib/python3.11/site-packages (from jupyter-core->jupyterlab) (4.2.0)\n", "Requirement already satisfied: babel>=2.10 in /usr/local/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab) (2.14.0)\n", "Requirement already satisfied: json5>=0.9.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab) (0.9.14)\n", "Requirement already satisfied: jsonschema>=4.18.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab) (4.21.1)\n", "Requirement already satisfied: requests>=2.31 in /usr/local/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab) (2.31.0)\n", "Requirement already satisfied: colorama in /usr/local/lib/python3.11/site-packages (from nbdime~=4.0.1->jupyterlab-git) (0.4.6)\n", "Requirement already satisfied: gitpython!=2.1.4,!=2.1.5,!=2.1.6 in /usr/local/lib/python3.11/site-packages (from nbdime~=4.0.1->jupyterlab-git) (3.1.41)\n", "Requirement already satisfied: jupyter-server-mathjax>=0.2.2 in /usr/local/lib/python3.11/site-packages (from nbdime~=4.0.1->jupyterlab-git) (0.2.6)\n", "Requirement already satisfied: pygments in /usr/local/lib/python3.11/site-packages (from nbdime~=4.0.1->jupyterlab-git) (2.17.2)\n", "Requirement already satisfied: fastjsonschema in /usr/local/lib/python3.11/site-packages (from nbformat->jupyterlab-git) (2.19.1)\n", "Requirement already satisfied: comm>=0.1.1 in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (0.2.1)\n", "Requirement already satisfied: debugpy>=1.6.5 in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (1.8.0)\n", "Requirement already satisfied: ipython>=7.23.1 in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (8.21.0)\n", "Requirement already satisfied: matplotlib-inline>=0.1 in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (0.1.6)\n", "Requirement already satisfied: nest-asyncio in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (1.6.0)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (5.9.8)\n", "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.11/site-packages (from pexpect->jupyterlab-git) (0.7.0)\n", "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.11/site-packages (from gitpython!=2.1.4,!=2.1.5,!=2.1.6->nbdime~=4.0.1->jupyterlab-git) (4.0.11)\n", "Requirement already satisfied: decorator in /usr/local/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel->jupyterlab) (5.1.1)\n", "Requirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel->jupyterlab) (0.19.1)\n", "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in /usr/local/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel->jupyterlab) (3.0.43)\n", "Requirement already satisfied: stack-data in /usr/local/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel->jupyterlab) (0.6.3)\n", "Requirement already satisfied: attrs>=22.2.0 in /usr/local/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.19.0->jupyterlab) (23.2.0)\n", "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.19.0->jupyterlab) (2023.12.1)\n", "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.19.0->jupyterlab) (0.33.0)\n", "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.19.0->jupyterlab) (0.17.1)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/site-packages (from jupyter-client>=7.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (2.8.2)\n", "Requirement already satisfied: python-json-logger>=2.0.4 in /usr/local/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (2.0.7)\n", "Requirement already satisfied: pyyaml>=5.3 in /usr/local/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (6.0.1)\n", "Requirement already satisfied: rfc3339-validator in /usr/local/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (0.1.4)\n", "Requirement already satisfied: rfc3986-validator>=0.1.1 in /usr/local/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (0.1.1)\n", "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (4.12.3)\n", "Requirement already satisfied: bleach!=5.0.0 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (6.1.0)\n", "Requirement already satisfied: defusedxml in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.7.1)\n", "Requirement already satisfied: jupyterlab-pygments in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.3.0)\n", "Requirement already satisfied: mistune<4,>=2.0.3 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (3.0.2)\n", "Requirement already satisfied: nbclient>=0.5.0 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.9.0)\n", "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (1.5.1)\n", "Requirement already satisfied: tinycss2 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (1.2.1)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/site-packages (from requests>=2.31->jupyterlab-server<3,>=2.19.0->jupyterlab) (2.1.1)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/site-packages (from requests>=2.31->jupyterlab-server<3,>=2.19.0->jupyterlab) (2.2.0)\n", "Requirement already satisfied: argon2-cffi-bindings in /usr/local/lib/python3.11/site-packages (from argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab) (21.2.0)\n", "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.11/site-packages (from bleach!=5.0.0->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (1.16.0)\n", "Requirement already satisfied: webencodings in /usr/local/lib/python3.11/site-packages (from bleach!=5.0.0->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.5.1)\n", "Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.11/site-packages (from gitdb<5,>=4.0.1->gitpython!=2.1.4,!=2.1.5,!=2.1.6->nbdime~=4.0.1->jupyterlab-git) (5.0.1)\n", "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /usr/local/lib/python3.11/site-packages (from jedi>=0.16->ipython>=7.23.1->ipykernel->jupyterlab) (0.8.3)\n", "Requirement already satisfied: fqdn in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.5.1)\n", "Requirement already satisfied: isoduration in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (20.11.0)\n", "Requirement already satisfied: jsonpointer>1.13 in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (2.4)\n", "Requirement already satisfied: uri-template in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.3.0)\n", "Requirement already satisfied: webcolors>=1.11 in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.13)\n", "Requirement already satisfied: wcwidth in /usr/local/lib/python3.11/site-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython>=7.23.1->ipykernel->jupyterlab) (0.2.13)\n", "Requirement already satisfied: cffi>=1.0.1 in /usr/local/lib/python3.11/site-packages (from argon2-cffi-bindings->argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab) (1.16.0)\n", "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.11/site-packages (from beautifulsoup4->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (2.5)\n", "Requirement already satisfied: executing>=1.2.0 in /usr/local/lib/python3.11/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyterlab) (2.0.1)\n", "Requirement already satisfied: asttokens>=2.1.0 in /usr/local/lib/python3.11/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyterlab) (2.4.1)\n", "Requirement already satisfied: pure-eval in /usr/local/lib/python3.11/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyterlab) (0.2.2)\n", "Requirement already satisfied: pycparser in /usr/local/lib/python3.11/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab) (2.21)\n", "Requirement already satisfied: arrow>=0.15.0 in /usr/local/lib/python3.11/site-packages (from isoduration->jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.3.0)\n", "Requirement already satisfied: types-python-dateutil>=2.8.10 in /usr/local/lib/python3.11/site-packages (from arrow>=0.15.0->isoduration->jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (2.8.19.20240106)\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ], "source": [ "pip install --upgrade jupyterlab jupyterlab-git" ] }, { "cell_type": "code", "execution_count": 22, "id": "05d58512-a9e2-4319-88bf-9331c6a0584c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", " _| _| _| _| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _|_|_|_| _|_| _|_|_| _|_|_|_|\n", " _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _|\n", " _|_|_|_| _| _| _| _|_| _| _|_| _| _| _| _| _| _|_| _|_|_| _|_|_|_| _| _|_|_|\n", " _| _| _| _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _|\n", " _| _| _|_| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _| _| _| _|_|_| _|_|_|_|\n", "\n", " To login, `huggingface_hub` requires a token generated from https://huggingface.co/settings/tokens .\n" ] }, { "name": "stdin", "output_type": "stream", "text": [ "Token: ········\n", "Add token as git credential? (Y/n) n\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Token is valid (permission: write).\n", "Your token has been saved to /root/.cache/huggingface/token\n", "Login successful\n" ] } ], "source": [ "from huggingface_hub import interpreter_login\n", "interpreter_login()" ] }, { "cell_type": "code", "execution_count": 49, "id": "bcb01d1b-9a48-46fe-b020-51a4d61df532", "metadata": { "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Collecting torch==2.1.0\n", " Downloading torch-2.1.0-cp311-cp311-manylinux1_x86_64.whl.metadata (25 kB)\n", "Collecting pytorch-lightning==1.9.4\n", " Downloading pytorch_lightning-1.9.4-py3-none-any.whl (827 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m827.8/827.8 kB\u001b[0m \u001b[31m120.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting accelerate==0.21.0\n", " Downloading accelerate-0.21.0-py3-none-any.whl.metadata (17 kB)\n", "Collecting tokenizers==0.13.3\n", " Downloading tokenizers-0.13.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m292.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: transformers in /usr/local/lib/python3.11/site-packages (4.37.2)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (3.13.1)\n", "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (4.9.0)\n", "Requirement already satisfied: sympy in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (1.12)\n", "Requirement already satisfied: networkx in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (3.2.1)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (3.1.3)\n", "Requirement already satisfied: fsspec in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (2023.10.0)\n", "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.105)\n", "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.105)\n", "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.105)\n", "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (8.9.2.26)\n", "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.3.1)\n", "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (11.0.2.54)\n", "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (10.3.2.106)\n", "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (11.4.5.107)\n", "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.0.106)\n", "Requirement already satisfied: nvidia-nccl-cu12==2.18.1 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (2.18.1)\n", "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.105)\n", "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (2.1.0)\n", "Requirement already satisfied: numpy>=1.17.2 in /usr/local/lib/python3.11/site-packages (from pytorch-lightning==1.9.4) (1.26.4)\n", "Requirement already satisfied: tqdm>=4.57.0 in /usr/local/lib/python3.11/site-packages (from pytorch-lightning==1.9.4) (4.66.2)\n", "Requirement already satisfied: PyYAML>=5.4 in /usr/local/lib/python3.11/site-packages (from pytorch-lightning==1.9.4) (6.0.1)\n", "Collecting torchmetrics>=0.7.0 (from pytorch-lightning==1.9.4)\n", " Downloading torchmetrics-1.3.1-py3-none-any.whl.metadata (19 kB)\n", "Requirement already satisfied: packaging>=17.1 in /usr/local/lib/python3.11/site-packages (from pytorch-lightning==1.9.4) (23.2)\n", "Collecting lightning-utilities>=0.6.0.post0 (from pytorch-lightning==1.9.4)\n", " Downloading lightning_utilities-0.10.1-py3-none-any.whl.metadata (4.8 kB)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.11/site-packages (from accelerate==0.21.0) (5.9.8)\n", "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.11/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch==2.1.0) (12.3.101)\n", "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /usr/local/lib/python3.11/site-packages (from transformers) (0.20.3)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.11/site-packages (from transformers) (2023.12.25)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.11/site-packages (from transformers) (2.31.0)\n", "INFO: pip is looking at multiple versions of transformers to determine which version is compatible with other requirements. This could take a while.\n", "Collecting transformers\n", " Downloading transformers-4.37.1-py3-none-any.whl.metadata (129 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.4/129.4 kB\u001b[0m \u001b[31m333.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.37.0-py3-none-any.whl.metadata (129 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.4/129.4 kB\u001b[0m \u001b[31m388.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.36.2-py3-none-any.whl.metadata (126 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.8/126.8 kB\u001b[0m \u001b[31m357.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.36.1-py3-none-any.whl.metadata (126 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.8/126.8 kB\u001b[0m \u001b[31m568.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.36.0-py3-none-any.whl.metadata (126 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.8/126.8 kB\u001b[0m \u001b[31m377.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.35.2-py3-none-any.whl.metadata (123 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m123.5/123.5 kB\u001b[0m \u001b[31m318.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.35.1-py3-none-any.whl.metadata (123 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m123.1/123.1 kB\u001b[0m \u001b[31m264.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hINFO: pip is still looking at multiple versions of transformers to determine which version is compatible with other requirements. This could take a while.\n", " Downloading transformers-4.35.0-py3-none-any.whl.metadata (123 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m123.1/123.1 kB\u001b[0m \u001b[31m373.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.34.1-py3-none-any.whl.metadata (121 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.5/121.5 kB\u001b[0m \u001b[31m373.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.34.0-py3-none-any.whl.metadata (121 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.5/121.5 kB\u001b[0m \u001b[31m370.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.33.3-py3-none-any.whl.metadata (119 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.9/119.9 kB\u001b[0m \u001b[31m323.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.11/site-packages (from transformers) (0.4.2)\n", "Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /usr/local/lib/python3.11/site-packages (from fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (3.8.3)\n", "Requirement already satisfied: setuptools in /usr/local/lib/python3.11/site-packages (from lightning-utilities>=0.6.0.post0->pytorch-lightning==1.9.4) (68.1.2)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/site-packages (from jinja2->torch==2.1.0) (2.1.5)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2.1.1)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2.2.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2024.2.2)\n", "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.11/site-packages (from sympy->torch==2.1.0) (1.3.0)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (23.2.0)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (6.0.5)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (4.0.3)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (1.9.4)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (1.4.1)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (1.3.1)\n", "Downloading torch-2.1.0-cp311-cp311-manylinux1_x86_64.whl (670.2 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m670.2/670.2 MB\u001b[0m \u001b[31m284.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading accelerate-0.21.0-py3-none-any.whl (244 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m244.2/244.2 kB\u001b[0m \u001b[31m504.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading transformers-4.33.3-py3-none-any.whl (7.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m116.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading lightning_utilities-0.10.1-py3-none-any.whl (24 kB)\n", "Downloading torchmetrics-1.3.1-py3-none-any.whl (840 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m840.4/840.4 kB\u001b[0m \u001b[31m646.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hInstalling collected packages: tokenizers, lightning-utilities, transformers, torch, torchmetrics, accelerate, pytorch-lightning\n", " Attempting uninstall: tokenizers\n", " Found existing installation: tokenizers 0.15.2\n", " Uninstalling tokenizers-0.15.2:\n", " Successfully uninstalled tokenizers-0.15.2\n", " Attempting uninstall: transformers\n", " Found existing installation: transformers 4.37.2\n", " Uninstalling transformers-4.37.2:\n", " Successfully uninstalled transformers-4.37.2\n", " Attempting uninstall: torch\n", " Found existing installation: torch 2.1.2\n", " Uninstalling torch-2.1.2:\n", " Successfully uninstalled torch-2.1.2\n", " Attempting uninstall: accelerate\n", " Found existing installation: accelerate 0.27.2\n", " Uninstalling accelerate-0.27.2:\n", " Successfully uninstalled accelerate-0.27.2\n", "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", "xformers 0.0.23.post1 requires torch==2.1.2, but you have torch 2.1.0 which is incompatible.\n", "vllm 0.3.0 requires torch==2.1.2, but you have torch 2.1.0 which is incompatible.\n", "vllm 0.3.0 requires transformers>=4.37.0, but you have transformers 4.33.3 which is incompatible.\u001b[0m\u001b[31m\n", "\u001b[0mSuccessfully installed accelerate-0.21.0 lightning-utilities-0.10.1 pytorch-lightning-1.9.4 tokenizers-0.13.3 torch-2.1.0 torchmetrics-1.3.1 transformers-4.33.3\n" ] } ], "source": [ " !pip install torch==2.1.0 pytorch-lightning==1.9.4 accelerate==0.21.0 tokenizers==0.13.3 transformers" ] }, { "cell_type": "code", "execution_count": 51, "id": "d76b4865-df75-4793-b8b9-d97523445945", "metadata": {}, "outputs": [], "source": [ "model.enable_input_require_grads()" ] }, { "cell_type": "code", "execution_count": null, "id": "3bf553b6-b26c-49c3-9407-74c8d53a395e", "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [120/200 01:14 < 00:50, 1.58 it/s, Epoch 0.03/1]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
StepTraining Loss
1001.069900

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "trainer = Trainer(\n", " model=model,\n", " train_dataset=tokenized_data,\n", " args=training_arguments,\n", " data_collator=DataCollatorForLanguageModeling(tokenizer, mlm=False)\n", ")\n", "trainer.train()" ] }, { "cell_type": "code", "execution_count": 35, "id": "263cc15e-8e9d-4bd8-9708-ec1638bc1165", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Loading checkpoint shards: 100%|██████████| 2/2 [00:03<00:00, 1.90s/it]\n", "adapter_config.json: 100%|██████████| 613/613 [00:00<00:00, 3.83MB/s]\n", "adapter_model.safetensors: 100%|██████████| 73.4M/73.4M [00:00<00:00, 110MB/s]\n" ] }, { "data": { "text/plain": [ "PhiForCausalLM(\n", " (model): PhiModel(\n", " (embed_tokens): Embedding(51200, 2560)\n", " (embed_dropout): Dropout(p=0.0, inplace=False)\n", " (layers): ModuleList(\n", " (0-31): 32 x PhiDecoderLayer(\n", " (self_attn): PhiAttention(\n", " (q_proj): Linear(in_features=2560, out_features=2560, bias=True)\n", " (k_proj): Linear(in_features=2560, out_features=2560, bias=True)\n", " (v_proj): Linear(in_features=2560, out_features=2560, bias=True)\n", " (dense): Linear(in_features=2560, out_features=2560, bias=True)\n", " (rotary_emb): PhiRotaryEmbedding()\n", " )\n", " (mlp): PhiMLP(\n", " (activation_fn): NewGELUActivation()\n", " (fc1): Linear(in_features=2560, out_features=10240, bias=True)\n", " (fc2): Linear(in_features=10240, out_features=2560, bias=True)\n", " )\n", " (input_layernorm): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n", " (resid_dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (final_layernorm): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n", " )\n", " (lm_head): Linear(in_features=2560, out_features=51200, bias=True)\n", ")" ] }, "execution_count": 35, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from peft import PeftModel\n", "from transformers import AutoModelForCausalLM\n", "import torch\n", "model = AutoModelForCausalLM.from_pretrained(\"microsoft/phi-2\", trust_remote_code=True, torch_dtype=torch.float32)\n", "peft_model = PeftModel.from_pretrained(model, \"aissatoubalde/lab\", from_transformers=True)\n", "model = peft_model.merge_and_unload()\n", "model" ] }, { "cell_type": "code", "execution_count": 36, "id": "8eef4f4f-52da-4ba9-8a22-2b7874420562", "metadata": { "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "README.md: 100%|██████████| 5.18k/5.18k [00:00<00:00, 17.1MB/s]\n", "model-00001-of-00003.safetensors: 0%| | 0.00/4.98G [00:00