{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "b3b92bc7-d105-405f-970d-804d298b9976", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting accelerate\n", " Downloading accelerate-0.27.2-py3-none-any.whl.metadata (18 kB)\n", "Collecting transformers\n", " Downloading transformers-4.37.2-py3-none-any.whl.metadata (129 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.4/129.4 kB\u001b[0m \u001b[31m41.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting einops\n", " Downloading einops-0.7.0-py3-none-any.whl.metadata (13 kB)\n", "Collecting datasets\n", " Downloading datasets-2.17.0-py3-none-any.whl.metadata (20 kB)\n", "Collecting peft\n", " Downloading peft-0.8.2-py3-none-any.whl.metadata (25 kB)\n", "Collecting bitsandbytes\n", " Downloading bitsandbytes-0.42.0-py3-none-any.whl.metadata (9.9 kB)\n", "Collecting torch\n", " Downloading torch-2.2.0-cp311-cp311-manylinux1_x86_64.whl.metadata (25 kB)\n", "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.11/site-packages (from accelerate) (1.26.4)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/site-packages (from accelerate) (23.2)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.11/site-packages (from accelerate) (5.9.8)\n", "Requirement already satisfied: pyyaml in /usr/local/lib/python3.11/site-packages (from accelerate) (6.0.1)\n", "Collecting huggingface-hub (from accelerate)\n", " Downloading huggingface_hub-0.20.3-py3-none-any.whl.metadata (12 kB)\n", "Collecting safetensors>=0.3.1 (from accelerate)\n", " Downloading safetensors-0.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.8 kB)\n", "Collecting filelock (from transformers)\n", " Downloading filelock-3.13.1-py3-none-any.whl.metadata (2.8 kB)\n", "Collecting regex!=2019.12.17 (from transformers)\n", " Downloading regex-2023.12.25-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (40 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.9/40.9 kB\u001b[0m \u001b[31m422.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.11/site-packages (from transformers) (2.31.0)\n", "Collecting tokenizers<0.19,>=0.14 (from transformers)\n", " Downloading tokenizers-0.15.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n", "Collecting tqdm>=4.27 (from transformers)\n", " Downloading tqdm-4.66.2-py3-none-any.whl.metadata (57 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.6/57.6 kB\u001b[0m \u001b[31m207.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting pyarrow>=12.0.0 (from datasets)\n", " Downloading pyarrow-15.0.0-cp311-cp311-manylinux_2_28_x86_64.whl.metadata (3.0 kB)\n", "Collecting pyarrow-hotfix (from datasets)\n", " Downloading pyarrow_hotfix-0.6-py3-none-any.whl.metadata (3.6 kB)\n", "Collecting dill<0.3.9,>=0.3.0 (from datasets)\n", " Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n", "Collecting pandas (from datasets)\n", " Downloading pandas-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (19 kB)\n", "Collecting xxhash (from datasets)\n", " Downloading xxhash-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n", "Collecting multiprocess (from datasets)\n", " Downloading multiprocess-0.70.16-py311-none-any.whl.metadata (7.2 kB)\n", "Collecting fsspec<=2023.10.0,>=2023.1.0 (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets)\n", " Downloading fsspec-2023.10.0-py3-none-any.whl.metadata (6.8 kB)\n", "Requirement already satisfied: aiohttp in /usr/local/lib/python3.11/site-packages (from datasets) (3.8.3)\n", "Collecting scipy (from bitsandbytes)\n", " Downloading scipy-1.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (60 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.4/60.4 kB\u001b[0m \u001b[31m480.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.11/site-packages (from torch) (4.9.0)\n", "Collecting sympy (from torch)\n", " Downloading sympy-1.12-py3-none-any.whl.metadata (12 kB)\n", "Collecting networkx (from torch)\n", " Downloading networkx-3.2.1-py3-none-any.whl.metadata (5.2 kB)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/site-packages (from torch) (3.1.3)\n", "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch)\n", " Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch)\n", " Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch)\n", " Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n", "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch)\n", " Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n", "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch)\n", " Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch)\n", " Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Collecting nvidia-curand-cu12==10.3.2.106 (from torch)\n", " Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch)\n", " Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n", "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch)\n", " Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n", "Collecting nvidia-nccl-cu12==2.19.3 (from torch)\n", " Downloading nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl.metadata (1.8 kB)\n", "Collecting nvidia-nvtx-cu12==12.1.105 (from torch)\n", " Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.7 kB)\n", "Collecting triton==2.2.0 (from torch)\n", " Downloading triton-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.4 kB)\n", "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch)\n", " Downloading nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (23.2.0)\n", "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (2.1.1)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (6.0.5)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (4.0.3)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (1.9.4)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (1.4.1)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (1.3.1)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2.2.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2024.2.2)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/site-packages (from jinja2->torch) (2.1.5)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/site-packages (from pandas->datasets) (2.8.2)\n", "Collecting pytz>=2020.1 (from pandas->datasets)\n", " Downloading pytz-2024.1-py2.py3-none-any.whl.metadata (22 kB)\n", "Collecting tzdata>=2022.7 (from pandas->datasets)\n", " Downloading tzdata-2024.1-py2.py3-none-any.whl.metadata (1.4 kB)\n", "Collecting mpmath>=0.19 (from sympy->torch)\n", " Downloading mpmath-1.3.0-py3-none-any.whl.metadata (8.6 kB)\n", "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n", "Downloading accelerate-0.27.2-py3-none-any.whl (279 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m280.0/280.0 kB\u001b[0m \u001b[31m11.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading transformers-4.37.2-py3-none-any.whl (8.4 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.4/8.4 MB\u001b[0m \u001b[31m285.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading einops-0.7.0-py3-none-any.whl (44 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.6/44.6 kB\u001b[0m \u001b[31m444.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading datasets-2.17.0-py3-none-any.whl (536 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m536.6/536.6 kB\u001b[0m \u001b[31m484.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading peft-0.8.2-py3-none-any.whl (183 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m183.4/183.4 kB\u001b[0m \u001b[31m480.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading bitsandbytes-0.42.0-py3-none-any.whl (105.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m105.0/105.0 MB\u001b[0m \u001b[31m180.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading torch-2.2.0-cp311-cp311-manylinux1_x86_64.whl (755.5 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m755.5/755.5 MB\u001b[0m \u001b[31m220.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m168.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m130.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m238.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m490.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m216.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m262.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m143.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m137.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m288.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m166.0/166.0 MB\u001b[0m \u001b[31m266.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m446.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading triton-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (167.9 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m167.9/167.9 MB\u001b[0m \u001b[31m235.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading dill-0.3.8-py3-none-any.whl (116 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m352.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading fsspec-2023.10.0-py3-none-any.whl (166 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m166.4/166.4 kB\u001b[0m \u001b[31m496.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading huggingface_hub-0.20.3-py3-none-any.whl (330 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m330.1/330.1 kB\u001b[0m \u001b[31m548.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading pyarrow-15.0.0-cp311-cp311-manylinux_2_28_x86_64.whl (38.3 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m38.3/38.3 MB\u001b[0m \u001b[31m164.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading regex-2023.12.25-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (785 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m785.1/785.1 kB\u001b[0m \u001b[31m540.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading safetensors-0.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m564.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading tokenizers-0.15.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.6/3.6 MB\u001b[0m \u001b[31m167.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading tqdm-4.66.2-py3-none-any.whl (78 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.3/78.3 kB\u001b[0m \u001b[31m271.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading filelock-3.13.1-py3-none-any.whl (11 kB)\n", "Downloading multiprocess-0.70.16-py311-none-any.whl (143 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.5/143.5 kB\u001b[0m \u001b[31m358.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading networkx-3.2.1-py3-none-any.whl (1.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m399.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading pandas-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (13.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.0/13.0 MB\u001b[0m \u001b[31m221.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading pyarrow_hotfix-0.6-py3-none-any.whl (7.9 kB)\n", "Downloading scipy-1.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (38.4 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m38.4/38.4 MB\u001b[0m \u001b[31m226.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading sympy-1.12-py3-none-any.whl (5.7 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.7/5.7 MB\u001b[0m \u001b[31m211.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading xxhash-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.8/194.8 kB\u001b[0m \u001b[31m550.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading mpmath-1.3.0-py3-none-any.whl (536 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m536.2/536.2 kB\u001b[0m \u001b[31m560.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading pytz-2024.1-py2.py3-none-any.whl (505 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m505.5/505.5 kB\u001b[0m \u001b[31m554.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading tzdata-2024.1-py2.py3-none-any.whl (345 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m345.4/345.4 kB\u001b[0m \u001b[31m514.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl (20.5 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m20.5/20.5 MB\u001b[0m \u001b[31m243.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hInstalling collected packages: pytz, mpmath, xxhash, tzdata, tqdm, sympy, scipy, safetensors, regex, pyarrow-hotfix, pyarrow, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, networkx, fsspec, filelock, einops, dill, triton, pandas, nvidia-cusparse-cu12, nvidia-cudnn-cu12, multiprocess, huggingface-hub, bitsandbytes, tokenizers, nvidia-cusolver-cu12, transformers, torch, datasets, accelerate, peft\n", "Successfully installed accelerate-0.27.2 bitsandbytes-0.42.0 datasets-2.17.0 dill-0.3.8 einops-0.7.0 filelock-3.13.1 fsspec-2023.10.0 huggingface-hub-0.20.3 mpmath-1.3.0 multiprocess-0.70.16 networkx-3.2.1 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.3.101 nvidia-nvtx-cu12-12.1.105 pandas-2.2.0 peft-0.8.2 pyarrow-15.0.0 pyarrow-hotfix-0.6 pytz-2024.1 regex-2023.12.25 safetensors-0.4.2 scipy-1.12.0 sympy-1.12 tokenizers-0.15.2 torch-2.2.0 tqdm-4.66.2 transformers-4.37.2 triton-2.2.0 tzdata-2024.1 xxhash-3.4.1\n" ] } ], "source": [ "!pip install accelerate transformers einops datasets peft bitsandbytes torch" ] }, { "cell_type": "code", "execution_count": 2, "id": "46a53303-b585-4b02-956f-4af173410e25", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] } ], "source": [ "import torch\n", "from datasets import load_dataset, Dataset\n", "from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, Trainer, DataCollatorForLanguageModeling, BitsAndBytesConfig\n", "from peft import LoraConfig, get_peft_model\n", "import os" ] }, { "cell_type": "code", "execution_count": 3, "id": "ffa3ee11-3929-4032-8758-322cda3a912e", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "tokenizer_config.json: 100%|██████████| 7.34k/7.34k [00:00<00:00, 35.6MB/s]\n", "vocab.json: 100%|██████████| 798k/798k [00:00<00:00, 34.3MB/s]\n", "merges.txt: 100%|██████████| 456k/456k [00:00<00:00, 22.9MB/s]\n", "tokenizer.json: 100%|██████████| 2.11M/2.11M [00:00<00:00, 44.2MB/s]\n", "added_tokens.json: 100%|██████████| 1.08k/1.08k [00:00<00:00, 9.74MB/s]\n", "special_tokens_map.json: 100%|██████████| 99.0/99.0 [00:00<00:00, 882kB/s]\n", "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] } ], "source": [ "tokenizer = AutoTokenizer.from_pretrained(\"microsoft/phi-2\", trust_remote_code=True)\n", "tokenizer.pad_token = tokenizer.eos_token" ] }, { "cell_type": "code", "execution_count": 4, "id": "b17096e6-c633-4d88-8cfc-fb738c1e4ca0", "metadata": { "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "config.json: 100%|██████████| 863/863 [00:00<00:00, 7.14MB/s]\n", "configuration_phi.py: 100%|██████████| 9.26k/9.26k [00:00<00:00, 28.2MB/s]\n", "A new version of the following files was downloaded from https://huggingface.co/microsoft/phi-2:\n", "- configuration_phi.py\n", ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n", "modeling_phi.py: 100%|██████████| 62.7k/62.7k [00:00<00:00, 157MB/s]\n", "A new version of the following files was downloaded from https://huggingface.co/microsoft/phi-2:\n", "- modeling_phi.py\n", ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n", "model.safetensors.index.json: 100%|██████████| 35.7k/35.7k [00:00<00:00, 118MB/s]\n", "Downloading shards: 0%| | 0/2 [00:00=1.19.5 in /usr/local/lib/python3.11/site-packages (from scikit-learn) (1.26.4)\n", "Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.11/site-packages (from scikit-learn) (1.12.0)\n", "Collecting joblib>=1.2.0 (from scikit-learn)\n", " Downloading joblib-1.3.2-py3-none-any.whl.metadata (5.4 kB)\n", "Collecting threadpoolctl>=2.0.0 (from scikit-learn)\n", " Downloading threadpoolctl-3.3.0-py3-none-any.whl.metadata (13 kB)\n", "Downloading scikit_learn-1.4.1.post1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.1 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.1/12.1 MB\u001b[0m \u001b[31m183.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading joblib-1.3.2-py3-none-any.whl (302 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m302.2/302.2 kB\u001b[0m \u001b[31m531.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading threadpoolctl-3.3.0-py3-none-any.whl (17 kB)\n", "Installing collected packages: threadpoolctl, joblib, scikit-learn\n", "Successfully installed joblib-1.3.2 scikit-learn-1.4.1.post1 threadpoolctl-3.3.0\n" ] } ], "source": [ "!pip install scikit-learn" ] }, { "cell_type": "code", "execution_count": 9, "id": "1ee7fd2a-38e4-4f23-a978-0bdeeda64d8b", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "\n", "from sklearn.model_selection import train_test_split\n", "dataset_name='data.csv'\n", "df = pd.read_csv(dataset_name)\n", "#train, test = train_test_split(df, test_size=0.2)" ] }, { "cell_type": "code", "execution_count": 10, "id": "e84c29e2-843e-42c2-8c0f-324d392e671c", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Tokenizing data: 100%|██████████| 16412/16412 [00:02<00:00, 6676.76 examples/s]\n" ] }, { "data": { "text/plain": [ "Dataset({\n", " features: ['input_ids', 'attention_mask'],\n", " num_rows: 16412\n", "})" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data_df = df\n", "data_df[\"text\"] = data_df[[\"user\", \"assistant\"]].apply(lambda x: \"question: \" + str(x[\"user\"]) + \" answer: \" + str(x[\"assistant\"]), axis=1)\n", "data = Dataset.from_pandas(data_df)\n", "tokenized_data = data.map(tokenize, batched=True, desc=\"Tokenizing data\", remove_columns=data.column_names)\n", "tokenized_data" ] }, { "cell_type": "code", "execution_count": 21, "id": "ac968254-5338-49df-950d-222b82647407", "metadata": {}, "outputs": [], "source": [ "training_arguments = TrainingArguments(\n", " output_dir=\".\",\n", " per_device_train_batch_size=4,\n", " gradient_accumulation_steps=1,\n", " learning_rate=2e-4,\n", " lr_scheduler_type=\"cosine\",\n", " save_strategy=\"epoch\",\n", " logging_steps=100,\n", " max_steps=2000,\n", " num_train_epochs=3,\n", " push_to_hub=True\n", " )" ] }, { "cell_type": "code", "execution_count": 12, "id": "729df2d0-0890-4ac4-adf3-c167a6e9669d", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: huggingface_hub in /usr/local/lib/python3.11/site-packages (0.20.3)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (3.13.1)\n", "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (2023.10.0)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (2.31.0)\n", "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (4.66.2)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (6.0.1)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (4.9.0)\n", "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (23.2)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/site-packages (from requests->huggingface_hub) (2.1.1)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/site-packages (from requests->huggingface_hub) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/site-packages (from requests->huggingface_hub) (2.2.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/site-packages (from requests->huggingface_hub) (2024.2.2)\n" ] } ], "source": [ "!pip install huggingface_hub" ] }, { "cell_type": "code", "execution_count": 13, "id": "525f81a4-eb92-466e-bb9c-cd63122231ab", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: jupyterlab in /usr/local/lib/python3.11/site-packages (4.1.0)\n", "Collecting jupyterlab\n", " Downloading jupyterlab-4.1.1-py3-none-any.whl.metadata (15 kB)\n", "Collecting jupyterlab-git\n", " Downloading jupyterlab_git-0.50.0-py3-none-any.whl.metadata (31 kB)\n", "Requirement already satisfied: async-lru>=1.0.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (2.0.4)\n", "Requirement already satisfied: httpx>=0.25.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (0.26.0)\n", "Requirement already satisfied: ipykernel in /usr/local/lib/python3.11/site-packages (from jupyterlab) (6.29.1)\n", "Requirement already satisfied: jinja2>=3.0.3 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (3.1.3)\n", "Requirement already satisfied: jupyter-core in /usr/local/lib/python3.11/site-packages (from jupyterlab) (5.7.1)\n", "Requirement already satisfied: jupyter-lsp>=2.0.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (2.2.2)\n", "Requirement already satisfied: jupyter-server<3,>=2.4.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (2.12.5)\n", "Requirement already satisfied: jupyterlab-server<3,>=2.19.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (2.25.2)\n", "Requirement already satisfied: notebook-shim>=0.2 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (0.2.3)\n", "Requirement already satisfied: packaging in /usr/local/lib/python3.11/site-packages (from jupyterlab) (23.2)\n", "Requirement already satisfied: tornado>=6.2.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (6.4)\n", "Requirement already satisfied: traitlets in /usr/local/lib/python3.11/site-packages (from jupyterlab) (5.14.1)\n", "Collecting nbdime~=4.0.1 (from jupyterlab-git)\n", " Downloading nbdime-4.0.1-py3-none-any.whl.metadata (9.5 kB)\n", "Requirement already satisfied: nbformat in /usr/local/lib/python3.11/site-packages (from jupyterlab-git) (5.9.2)\n", "Requirement already satisfied: pexpect in /usr/local/lib/python3.11/site-packages (from jupyterlab-git) (4.9.0)\n", "Requirement already satisfied: anyio in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (4.2.0)\n", "Requirement already satisfied: certifi in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (2024.2.2)\n", "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (1.0.2)\n", "Requirement already satisfied: idna in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (3.6)\n", "Requirement already satisfied: sniffio in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (1.3.0)\n", "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.11/site-packages (from httpcore==1.*->httpx>=0.25.0->jupyterlab) (0.14.0)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/site-packages (from jinja2>=3.0.3->jupyterlab) (2.1.5)\n", "Requirement already satisfied: argon2-cffi in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (23.1.0)\n", "Requirement already satisfied: jupyter-client>=7.4.4 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (8.6.0)\n", "Requirement already satisfied: jupyter-events>=0.9.0 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.9.0)\n", "Requirement already satisfied: jupyter-server-terminals in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.5.2)\n", "Requirement already satisfied: nbconvert>=6.4.4 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (7.15.0)\n", "Requirement already satisfied: overrides in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (7.7.0)\n", "Requirement already satisfied: prometheus-client in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.19.0)\n", "Requirement already satisfied: pyzmq>=24 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (25.1.2)\n", "Requirement already satisfied: send2trash>=1.8.2 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (1.8.2)\n", "Requirement already satisfied: terminado>=0.8.3 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.18.0)\n", "Requirement already satisfied: websocket-client in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (1.7.0)\n", "Requirement already satisfied: platformdirs>=2.5 in /usr/local/lib/python3.11/site-packages (from jupyter-core->jupyterlab) (4.2.0)\n", "Requirement already satisfied: babel>=2.10 in /usr/local/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab) (2.14.0)\n", "Requirement already satisfied: json5>=0.9.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab) (0.9.14)\n", "Requirement already satisfied: jsonschema>=4.18.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab) (4.21.1)\n", "Requirement already satisfied: requests>=2.31 in /usr/local/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab) (2.31.0)\n", "Collecting colorama (from nbdime~=4.0.1->jupyterlab-git)\n", " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", "Collecting gitpython!=2.1.4,!=2.1.5,!=2.1.6 (from nbdime~=4.0.1->jupyterlab-git)\n", " Downloading GitPython-3.1.42-py3-none-any.whl.metadata (12 kB)\n", "Collecting jupyter-server-mathjax>=0.2.2 (from nbdime~=4.0.1->jupyterlab-git)\n", " Downloading jupyter_server_mathjax-0.2.6-py3-none-any.whl (3.1 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m110.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: pygments in /usr/local/lib/python3.11/site-packages (from nbdime~=4.0.1->jupyterlab-git) (2.17.2)\n", "Requirement already satisfied: fastjsonschema in /usr/local/lib/python3.11/site-packages (from nbformat->jupyterlab-git) (2.19.1)\n", "Requirement already satisfied: comm>=0.1.1 in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (0.2.1)\n", "Requirement already satisfied: debugpy>=1.6.5 in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (1.8.0)\n", "Requirement already satisfied: ipython>=7.23.1 in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (8.21.0)\n", "Requirement already satisfied: matplotlib-inline>=0.1 in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (0.1.6)\n", "Requirement already satisfied: nest-asyncio in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (1.6.0)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (5.9.8)\n", "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.11/site-packages (from pexpect->jupyterlab-git) (0.7.0)\n", "Collecting gitdb<5,>=4.0.1 (from gitpython!=2.1.4,!=2.1.5,!=2.1.6->nbdime~=4.0.1->jupyterlab-git)\n", " Downloading gitdb-4.0.11-py3-none-any.whl.metadata (1.2 kB)\n", "Requirement already satisfied: decorator in /usr/local/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel->jupyterlab) (5.1.1)\n", "Requirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel->jupyterlab) (0.19.1)\n", "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in /usr/local/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel->jupyterlab) (3.0.43)\n", "Requirement already satisfied: stack-data in /usr/local/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel->jupyterlab) (0.6.3)\n", "Requirement already satisfied: attrs>=22.2.0 in /usr/local/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.19.0->jupyterlab) (23.2.0)\n", "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.19.0->jupyterlab) (2023.12.1)\n", "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.19.0->jupyterlab) (0.33.0)\n", "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.19.0->jupyterlab) (0.17.1)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/site-packages (from jupyter-client>=7.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (2.8.2)\n", "Requirement already satisfied: python-json-logger>=2.0.4 in /usr/local/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (2.0.7)\n", "Requirement already satisfied: pyyaml>=5.3 in /usr/local/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (6.0.1)\n", "Requirement already satisfied: rfc3339-validator in /usr/local/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (0.1.4)\n", "Requirement already satisfied: rfc3986-validator>=0.1.1 in /usr/local/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (0.1.1)\n", "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (4.12.3)\n", "Requirement already satisfied: bleach!=5.0.0 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (6.1.0)\n", "Requirement already satisfied: defusedxml in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.7.1)\n", "Requirement already satisfied: jupyterlab-pygments in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.3.0)\n", "Requirement already satisfied: mistune<4,>=2.0.3 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (3.0.2)\n", "Requirement already satisfied: nbclient>=0.5.0 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.9.0)\n", "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (1.5.1)\n", "Requirement already satisfied: tinycss2 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (1.2.1)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/site-packages (from requests>=2.31->jupyterlab-server<3,>=2.19.0->jupyterlab) (2.1.1)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/site-packages (from requests>=2.31->jupyterlab-server<3,>=2.19.0->jupyterlab) (2.2.0)\n", "Requirement already satisfied: argon2-cffi-bindings in /usr/local/lib/python3.11/site-packages (from argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab) (21.2.0)\n", "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.11/site-packages (from bleach!=5.0.0->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (1.16.0)\n", "Requirement already satisfied: webencodings in /usr/local/lib/python3.11/site-packages (from bleach!=5.0.0->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.5.1)\n", "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->gitpython!=2.1.4,!=2.1.5,!=2.1.6->nbdime~=4.0.1->jupyterlab-git)\n", " Downloading smmap-5.0.1-py3-none-any.whl.metadata (4.3 kB)\n", "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /usr/local/lib/python3.11/site-packages (from jedi>=0.16->ipython>=7.23.1->ipykernel->jupyterlab) (0.8.3)\n", "Requirement already satisfied: fqdn in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.5.1)\n", "Requirement already satisfied: isoduration in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (20.11.0)\n", "Requirement already satisfied: jsonpointer>1.13 in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (2.4)\n", "Requirement already satisfied: uri-template in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.3.0)\n", "Requirement already satisfied: webcolors>=1.11 in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.13)\n", "Requirement already satisfied: wcwidth in /usr/local/lib/python3.11/site-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython>=7.23.1->ipykernel->jupyterlab) (0.2.13)\n", "Requirement already satisfied: cffi>=1.0.1 in /usr/local/lib/python3.11/site-packages (from argon2-cffi-bindings->argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab) (1.16.0)\n", "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.11/site-packages (from beautifulsoup4->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (2.5)\n", "Requirement already satisfied: executing>=1.2.0 in /usr/local/lib/python3.11/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyterlab) (2.0.1)\n", "Requirement already satisfied: asttokens>=2.1.0 in /usr/local/lib/python3.11/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyterlab) (2.4.1)\n", "Requirement already satisfied: pure-eval in /usr/local/lib/python3.11/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyterlab) (0.2.2)\n", "Requirement already satisfied: pycparser in /usr/local/lib/python3.11/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab) (2.21)\n", "Requirement already satisfied: arrow>=0.15.0 in /usr/local/lib/python3.11/site-packages (from isoduration->jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.3.0)\n", "Requirement already satisfied: types-python-dateutil>=2.8.10 in /usr/local/lib/python3.11/site-packages (from arrow>=0.15.0->isoduration->jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (2.8.19.20240106)\n", "Downloading jupyterlab-4.1.1-py3-none-any.whl (11.4 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.4/11.4 MB\u001b[0m \u001b[31m474.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading jupyterlab_git-0.50.0-py3-none-any.whl (1.2 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m548.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading nbdime-4.0.1-py3-none-any.whl (5.9 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.9/5.9 MB\u001b[0m \u001b[31m217.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading GitPython-3.1.42-py3-none-any.whl (195 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m195.4/195.4 kB\u001b[0m \u001b[31m402.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading gitdb-4.0.11-py3-none-any.whl (62 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m339.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading smmap-5.0.1-py3-none-any.whl (24 kB)\n", "Installing collected packages: smmap, colorama, gitdb, gitpython, jupyter-server-mathjax, nbdime, jupyterlab, jupyterlab-git\n", " Attempting uninstall: jupyterlab\n", " Found existing installation: jupyterlab 4.1.0\n", " Uninstalling jupyterlab-4.1.0:\n", " Successfully uninstalled jupyterlab-4.1.0\n", "Successfully installed colorama-0.4.6 gitdb-4.0.11 gitpython-3.1.42 jupyter-server-mathjax-0.2.6 jupyterlab-4.1.1 jupyterlab-git-0.50.0 nbdime-4.0.1 smmap-5.0.1\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ], "source": [ "pip install --upgrade jupyterlab jupyterlab-git" ] }, { "cell_type": "code", "execution_count": 14, "id": "05d58512-a9e2-4319-88bf-9331c6a0584c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", " _| _| _| _| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _|_|_|_| _|_| _|_|_| _|_|_|_|\n", " _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _|\n", " _|_|_|_| _| _| _| _|_| _| _|_| _| _| _| _| _| _|_| _|_|_| _|_|_|_| _| _|_|_|\n", " _| _| _| _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _|\n", " _| _| _|_| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _| _| _| _|_|_| _|_|_|_|\n", "\n", " To login, `huggingface_hub` requires a token generated from https://huggingface.co/settings/tokens .\n" ] }, { "name": "stdin", "output_type": "stream", "text": [ "Token: ········\n", "Add token as git credential? (Y/n) n\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Token is valid (permission: write).\n", "Your token has been saved to /root/.cache/huggingface/token\n", "Login successful\n" ] } ], "source": [ "from huggingface_hub import interpreter_login\n", "interpreter_login()" ] }, { "cell_type": "code", "execution_count": 15, "id": "bcb01d1b-9a48-46fe-b020-51a4d61df532", "metadata": { "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Collecting torch==2.1.0\n", " Downloading torch-2.1.0-cp311-cp311-manylinux1_x86_64.whl.metadata (25 kB)\n", "Collecting pytorch-lightning==1.9.4\n", " Downloading pytorch_lightning-1.9.4-py3-none-any.whl.metadata (22 kB)\n", "Collecting accelerate==0.21.0\n", " Downloading accelerate-0.21.0-py3-none-any.whl.metadata (17 kB)\n", "Collecting tokenizers==0.13.3\n", " Downloading tokenizers-0.13.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n", "Requirement already satisfied: transformers in /usr/local/lib/python3.11/site-packages (4.37.2)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (3.13.1)\n", "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (4.9.0)\n", "Requirement already satisfied: sympy in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (1.12)\n", "Requirement already satisfied: networkx in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (3.2.1)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (3.1.3)\n", "Requirement already satisfied: fsspec in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (2023.10.0)\n", "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.105)\n", "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.105)\n", "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.105)\n", "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (8.9.2.26)\n", "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.3.1)\n", "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (11.0.2.54)\n", "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (10.3.2.106)\n", "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (11.4.5.107)\n", "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.0.106)\n", "Collecting nvidia-nccl-cu12==2.18.1 (from torch==2.1.0)\n", " Downloading nvidia_nccl_cu12-2.18.1-py3-none-manylinux1_x86_64.whl.metadata (1.8 kB)\n", "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.105)\n", "Collecting triton==2.1.0 (from torch==2.1.0)\n", " Downloading triton-2.1.0-0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.3 kB)\n", "Requirement already satisfied: numpy>=1.17.2 in /usr/local/lib/python3.11/site-packages (from pytorch-lightning==1.9.4) (1.26.4)\n", "Requirement already satisfied: tqdm>=4.57.0 in /usr/local/lib/python3.11/site-packages (from pytorch-lightning==1.9.4) (4.66.2)\n", "Requirement already satisfied: PyYAML>=5.4 in /usr/local/lib/python3.11/site-packages (from pytorch-lightning==1.9.4) (6.0.1)\n", "Collecting torchmetrics>=0.7.0 (from pytorch-lightning==1.9.4)\n", " Downloading torchmetrics-1.3.1-py3-none-any.whl.metadata (19 kB)\n", "Requirement already satisfied: packaging>=17.1 in /usr/local/lib/python3.11/site-packages (from pytorch-lightning==1.9.4) (23.2)\n", "Collecting lightning-utilities>=0.6.0.post0 (from pytorch-lightning==1.9.4)\n", " Downloading lightning_utilities-0.10.1-py3-none-any.whl.metadata (4.8 kB)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.11/site-packages (from accelerate==0.21.0) (5.9.8)\n", "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.11/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch==2.1.0) (12.3.101)\n", "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /usr/local/lib/python3.11/site-packages (from transformers) (0.20.3)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.11/site-packages (from transformers) (2023.12.25)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.11/site-packages (from transformers) (2.31.0)\n", "INFO: pip is looking at multiple versions of transformers to determine which version is compatible with other requirements. This could take a while.\n", "Collecting transformers\n", " Downloading transformers-4.37.1-py3-none-any.whl.metadata (129 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.4/129.4 kB\u001b[0m \u001b[31m391.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.37.0-py3-none-any.whl.metadata (129 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.4/129.4 kB\u001b[0m \u001b[31m500.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.36.2-py3-none-any.whl.metadata (126 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.8/126.8 kB\u001b[0m \u001b[31m407.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.36.1-py3-none-any.whl.metadata (126 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.8/126.8 kB\u001b[0m \u001b[31m421.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.36.0-py3-none-any.whl.metadata (126 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.8/126.8 kB\u001b[0m \u001b[31m467.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.35.2-py3-none-any.whl.metadata (123 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m123.5/123.5 kB\u001b[0m \u001b[31m445.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.35.1-py3-none-any.whl.metadata (123 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m123.1/123.1 kB\u001b[0m \u001b[31m279.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hINFO: pip is still looking at multiple versions of transformers to determine which version is compatible with other requirements. This could take a while.\n", " Downloading transformers-4.35.0-py3-none-any.whl.metadata (123 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m123.1/123.1 kB\u001b[0m \u001b[31m312.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.34.1-py3-none-any.whl.metadata (121 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.5/121.5 kB\u001b[0m \u001b[31m331.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.34.0-py3-none-any.whl.metadata (121 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.5/121.5 kB\u001b[0m \u001b[31m243.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.33.3-py3-none-any.whl.metadata (119 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.9/119.9 kB\u001b[0m \u001b[31m411.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.11/site-packages (from transformers) (0.4.2)\n", "Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /usr/local/lib/python3.11/site-packages (from fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (3.8.3)\n", "Requirement already satisfied: setuptools in /usr/local/lib/python3.11/site-packages (from lightning-utilities>=0.6.0.post0->pytorch-lightning==1.9.4) (68.1.2)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/site-packages (from jinja2->torch==2.1.0) (2.1.5)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2.1.1)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2.2.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2024.2.2)\n", "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.11/site-packages (from sympy->torch==2.1.0) (1.3.0)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (23.2.0)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (6.0.5)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (4.0.3)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (1.9.4)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (1.4.1)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (1.3.1)\n", "Downloading torch-2.1.0-cp311-cp311-manylinux1_x86_64.whl (670.2 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m670.2/670.2 MB\u001b[0m \u001b[31m135.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading pytorch_lightning-1.9.4-py3-none-any.whl (827 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m827.8/827.8 kB\u001b[0m \u001b[31m549.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading accelerate-0.21.0-py3-none-any.whl (244 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m244.2/244.2 kB\u001b[0m \u001b[31m480.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading tokenizers-0.13.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m189.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_nccl_cu12-2.18.1-py3-none-manylinux1_x86_64.whl (209.8 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m209.8/209.8 MB\u001b[0m \u001b[31m244.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading triton-2.1.0-0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (89.2 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m89.2/89.2 MB\u001b[0m \u001b[31m150.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading transformers-4.33.3-py3-none-any.whl (7.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m234.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading lightning_utilities-0.10.1-py3-none-any.whl (24 kB)\n", "Downloading torchmetrics-1.3.1-py3-none-any.whl (840 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m840.4/840.4 kB\u001b[0m \u001b[31m545.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hInstalling collected packages: tokenizers, triton, nvidia-nccl-cu12, lightning-utilities, transformers, torch, torchmetrics, accelerate, pytorch-lightning\n", " Attempting uninstall: tokenizers\n", " Found existing installation: tokenizers 0.15.2\n", " Uninstalling tokenizers-0.15.2:\n", " Successfully uninstalled tokenizers-0.15.2\n", " Attempting uninstall: triton\n", " Found existing installation: triton 2.2.0\n", " Uninstalling triton-2.2.0:\n", " Successfully uninstalled triton-2.2.0\n", " Attempting uninstall: nvidia-nccl-cu12\n", " Found existing installation: nvidia-nccl-cu12 2.19.3\n", " Uninstalling nvidia-nccl-cu12-2.19.3:\n", " Successfully uninstalled nvidia-nccl-cu12-2.19.3\n", " Attempting uninstall: transformers\n", " Found existing installation: transformers 4.37.2\n", " Uninstalling transformers-4.37.2:\n", " Successfully uninstalled transformers-4.37.2\n", " Attempting uninstall: torch\n", " Found existing installation: torch 2.2.0\n", " Uninstalling torch-2.2.0:\n", " Successfully uninstalled torch-2.2.0\n", " Attempting uninstall: accelerate\n", " Found existing installation: accelerate 0.27.2\n", " Uninstalling accelerate-0.27.2:\n", " Successfully uninstalled accelerate-0.27.2\n", "Successfully installed accelerate-0.21.0 lightning-utilities-0.10.1 nvidia-nccl-cu12-2.18.1 pytorch-lightning-1.9.4 tokenizers-0.13.3 torch-2.1.0 torchmetrics-1.3.1 transformers-4.33.3 triton-2.1.0\n" ] } ], "source": [ " !pip install torch==2.1.0 pytorch-lightning==1.9.4 accelerate==0.21.0 tokenizers==0.13.3 transformers" ] }, { "cell_type": "code", "execution_count": 22, "id": "d76b4865-df75-4793-b8b9-d97523445945", "metadata": {}, "outputs": [], "source": [ "model.enable_input_require_grads()" ] }, { "cell_type": "code", "execution_count": null, "id": "3bf553b6-b26c-49c3-9407-74c8d53a395e", "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [1991/2000 07:21 < 00:01, 4.50 it/s, Epoch 0.49/1]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
StepTraining Loss
1001.383200
2001.200800
3001.195300
4001.124500
5001.138700
6001.113100
7001.146400
8001.128400
9001.086100
10001.096400
11001.111300
12001.108000
13001.124300
14001.106300
15001.100400
16001.142600
17001.081100
18001.090200
19001.111600

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "trainer = Trainer(\n", " model=model,\n", " train_dataset=tokenized_data,\n", " args=training_arguments,\n", " data_collator=DataCollatorForLanguageModeling(tokenizer, mlm=False)\n", ")\n", "trainer.train()" ] }, { "cell_type": "code", "execution_count": null, "id": "263cc15e-8e9d-4bd8-9708-ec1638bc1165", "metadata": {}, "outputs": [], "source": [ "from peft import PeftModel\n", "from transformers import AutoModelForCausalLM\n", "import torch\n", "model = AutoModelForCausalLM.from_pretrained(\"microsoft/phi-2\", trust_remote_code=True, torch_dtype=torch.float32)\n", "peft_model = PeftModel.from_pretrained(model, \"aissatoubalde/lab\", from_transformers=True)\n", "model = peft_model.merge_and_unload()\n", "model" ] }, { "cell_type": "code", "execution_count": null, "id": "8eef4f4f-52da-4ba9-8a22-2b7874420562", "metadata": { "scrolled": true }, "outputs": [], "source": [ "model.push_to_hub('aissatoubalde/lab')" ] }, { "cell_type": "code", "execution_count": null, "id": "fa43c8e1-945e-41ef-b2f6-e90db1f50140", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.5" } }, "nbformat": 4, "nbformat_minor": 5 }