{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "b3b92bc7-d105-405f-970d-804d298b9976", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting accelerate\n", " Downloading accelerate-0.27.2-py3-none-any.whl.metadata (18 kB)\n", "Collecting transformers\n", " Downloading transformers-4.37.2-py3-none-any.whl.metadata (129 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.4/129.4 kB\u001b[0m \u001b[31m65.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting einops\n", " Downloading einops-0.7.0-py3-none-any.whl.metadata (13 kB)\n", "Collecting datasets\n", " Downloading datasets-2.17.1-py3-none-any.whl.metadata (20 kB)\n", "Collecting peft\n", " Downloading peft-0.8.2-py3-none-any.whl.metadata (25 kB)\n", "Collecting bitsandbytes\n", " Downloading bitsandbytes-0.42.0-py3-none-any.whl.metadata (9.9 kB)\n", "Collecting torch\n", " Downloading torch-2.2.0-cp311-cp311-manylinux1_x86_64.whl.metadata (25 kB)\n", "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.11/site-packages (from accelerate) (1.26.4)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/site-packages (from accelerate) (23.2)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.11/site-packages (from accelerate) (5.9.8)\n", "Requirement already satisfied: pyyaml in /usr/local/lib/python3.11/site-packages (from accelerate) (6.0.1)\n", "Collecting huggingface-hub (from accelerate)\n", " Downloading huggingface_hub-0.20.3-py3-none-any.whl.metadata (12 kB)\n", "Collecting safetensors>=0.3.1 (from accelerate)\n", " Downloading safetensors-0.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.8 kB)\n", "Collecting filelock (from transformers)\n", " Downloading filelock-3.13.1-py3-none-any.whl.metadata (2.8 kB)\n", "Collecting regex!=2019.12.17 (from transformers)\n", " Downloading regex-2023.12.25-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (40 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.9/40.9 kB\u001b[0m \u001b[31m450.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.11/site-packages (from transformers) (2.31.0)\n", "Collecting tokenizers<0.19,>=0.14 (from transformers)\n", " Downloading tokenizers-0.15.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n", "Collecting tqdm>=4.27 (from transformers)\n", " Downloading tqdm-4.66.2-py3-none-any.whl.metadata (57 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.6/57.6 kB\u001b[0m \u001b[31m130.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting pyarrow>=12.0.0 (from datasets)\n", " Downloading pyarrow-15.0.0-cp311-cp311-manylinux_2_28_x86_64.whl.metadata (3.0 kB)\n", "Collecting pyarrow-hotfix (from datasets)\n", " Downloading pyarrow_hotfix-0.6-py3-none-any.whl.metadata (3.6 kB)\n", "Collecting dill<0.3.9,>=0.3.0 (from datasets)\n", " Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n", "Collecting pandas (from datasets)\n", " Downloading pandas-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (19 kB)\n", "Collecting xxhash (from datasets)\n", " Downloading xxhash-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n", "Collecting multiprocess (from datasets)\n", " Downloading multiprocess-0.70.16-py311-none-any.whl.metadata (7.2 kB)\n", "Collecting fsspec<=2023.10.0,>=2023.1.0 (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets)\n", " Downloading fsspec-2023.10.0-py3-none-any.whl.metadata (6.8 kB)\n", "Requirement already satisfied: aiohttp in /usr/local/lib/python3.11/site-packages (from datasets) (3.8.3)\n", "Collecting scipy (from bitsandbytes)\n", " Downloading scipy-1.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (60 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.4/60.4 kB\u001b[0m \u001b[31m433.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.11/site-packages (from torch) (4.9.0)\n", "Collecting sympy (from torch)\n", " Downloading sympy-1.12-py3-none-any.whl.metadata (12 kB)\n", "Collecting networkx (from torch)\n", " Downloading networkx-3.2.1-py3-none-any.whl.metadata (5.2 kB)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/site-packages (from torch) (3.1.3)\n", "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch)\n", " Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch)\n", " Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch)\n", " Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n", "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch)\n", " Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n", "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch)\n", " Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch)\n", " Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Collecting nvidia-curand-cu12==10.3.2.106 (from torch)\n", " Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch)\n", " Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n", "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch)\n", " Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n", "Collecting nvidia-nccl-cu12==2.19.3 (from torch)\n", " Downloading nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl.metadata (1.8 kB)\n", "Collecting nvidia-nvtx-cu12==12.1.105 (from torch)\n", " Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.7 kB)\n", "Collecting triton==2.2.0 (from torch)\n", " Downloading triton-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.4 kB)\n", "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch)\n", " Downloading nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (23.2.0)\n", "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (2.1.1)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (6.0.5)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (4.0.3)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (1.9.4)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (1.4.1)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (1.3.1)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2.2.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2024.2.2)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/site-packages (from jinja2->torch) (2.1.5)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/site-packages (from pandas->datasets) (2.8.2)\n", "Collecting pytz>=2020.1 (from pandas->datasets)\n", " Downloading pytz-2024.1-py2.py3-none-any.whl.metadata (22 kB)\n", "Collecting tzdata>=2022.7 (from pandas->datasets)\n", " Downloading tzdata-2024.1-py2.py3-none-any.whl.metadata (1.4 kB)\n", "Collecting mpmath>=0.19 (from sympy->torch)\n", " Downloading mpmath-1.3.0-py3-none-any.whl.metadata (8.6 kB)\n", "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n", "Downloading accelerate-0.27.2-py3-none-any.whl (279 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m280.0/280.0 kB\u001b[0m \u001b[31m147.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading transformers-4.37.2-py3-none-any.whl (8.4 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.4/8.4 MB\u001b[0m \u001b[31m311.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading einops-0.7.0-py3-none-any.whl (44 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.6/44.6 kB\u001b[0m \u001b[31m441.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading datasets-2.17.1-py3-none-any.whl (536 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m536.7/536.7 kB\u001b[0m \u001b[31m504.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading peft-0.8.2-py3-none-any.whl (183 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m183.4/183.4 kB\u001b[0m \u001b[31m438.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading bitsandbytes-0.42.0-py3-none-any.whl (105.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m105.0/105.0 MB\u001b[0m \u001b[31m274.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading torch-2.2.0-cp311-cp311-manylinux1_x86_64.whl (755.5 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m755.5/755.5 MB\u001b[0m \u001b[31m235.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m255.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m220.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m200.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m566.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m272.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m277.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m280.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m303.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m315.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m166.0/166.0 MB\u001b[0m \u001b[31m258.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m194.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading triton-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (167.9 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m167.9/167.9 MB\u001b[0m \u001b[31m222.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading dill-0.3.8-py3-none-any.whl (116 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m147.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading fsspec-2023.10.0-py3-none-any.whl (166 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m166.4/166.4 kB\u001b[0m \u001b[31m221.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading huggingface_hub-0.20.3-py3-none-any.whl (330 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m330.1/330.1 kB\u001b[0m \u001b[31m248.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading pyarrow-15.0.0-cp311-cp311-manylinux_2_28_x86_64.whl (38.3 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m38.3/38.3 MB\u001b[0m \u001b[31m295.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading regex-2023.12.25-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (785 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m785.1/785.1 kB\u001b[0m \u001b[31m452.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading safetensors-0.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m456.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading tokenizers-0.15.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.6/3.6 MB\u001b[0m \u001b[31m371.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading tqdm-4.66.2-py3-none-any.whl (78 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.3/78.3 kB\u001b[0m \u001b[31m181.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading filelock-3.13.1-py3-none-any.whl (11 kB)\n", "Downloading multiprocess-0.70.16-py311-none-any.whl (143 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.5/143.5 kB\u001b[0m \u001b[31m368.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading networkx-3.2.1-py3-none-any.whl (1.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m411.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading pandas-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (13.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.0/13.0 MB\u001b[0m \u001b[31m310.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading pyarrow_hotfix-0.6-py3-none-any.whl (7.9 kB)\n", "Downloading scipy-1.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (38.4 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m38.4/38.4 MB\u001b[0m \u001b[31m195.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading sympy-1.12-py3-none-any.whl (5.7 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.7/5.7 MB\u001b[0m \u001b[31m308.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading xxhash-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.8/194.8 kB\u001b[0m \u001b[31m280.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading mpmath-1.3.0-py3-none-any.whl (536 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m536.2/536.2 kB\u001b[0m \u001b[31m399.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading pytz-2024.1-py2.py3-none-any.whl (505 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m505.5/505.5 kB\u001b[0m \u001b[31m493.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading tzdata-2024.1-py2.py3-none-any.whl (345 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m345.4/345.4 kB\u001b[0m \u001b[31m327.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl (20.5 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m20.5/20.5 MB\u001b[0m \u001b[31m280.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hInstalling collected packages: pytz, mpmath, xxhash, tzdata, tqdm, sympy, scipy, safetensors, regex, pyarrow-hotfix, pyarrow, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, networkx, fsspec, filelock, einops, dill, triton, pandas, nvidia-cusparse-cu12, nvidia-cudnn-cu12, multiprocess, huggingface-hub, bitsandbytes, tokenizers, nvidia-cusolver-cu12, transformers, torch, datasets, accelerate, peft\n", "Successfully installed accelerate-0.27.2 bitsandbytes-0.42.0 datasets-2.17.1 dill-0.3.8 einops-0.7.0 filelock-3.13.1 fsspec-2023.10.0 huggingface-hub-0.20.3 mpmath-1.3.0 multiprocess-0.70.16 networkx-3.2.1 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.3.101 nvidia-nvtx-cu12-12.1.105 pandas-2.2.0 peft-0.8.2 pyarrow-15.0.0 pyarrow-hotfix-0.6 pytz-2024.1 regex-2023.12.25 safetensors-0.4.2 scipy-1.12.0 sympy-1.12 tokenizers-0.15.2 torch-2.2.0 tqdm-4.66.2 transformers-4.37.2 triton-2.2.0 tzdata-2024.1 xxhash-3.4.1\n" ] } ], "source": [ "!pip install accelerate transformers einops datasets peft bitsandbytes torch" ] }, { "cell_type": "code", "execution_count": 2, "id": "46a53303-b585-4b02-956f-4af173410e25", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] } ], "source": [ "import torch\n", "from datasets import load_dataset, Dataset\n", "from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, Trainer, DataCollatorForLanguageModeling, BitsAndBytesConfig\n", "from peft import LoraConfig, get_peft_model\n", "import os" ] }, { "cell_type": "code", "execution_count": 3, "id": "ffa3ee11-3929-4032-8758-322cda3a912e", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "tokenizer_config.json: 100%|██████████| 7.34k/7.34k [00:00<00:00, 23.2MB/s]\n", "vocab.json: 100%|██████████| 798k/798k [00:00<00:00, 28.9MB/s]\n", "merges.txt: 100%|██████████| 456k/456k [00:00<00:00, 32.2MB/s]\n", "tokenizer.json: 100%|██████████| 2.11M/2.11M [00:00<00:00, 31.8MB/s]\n", "added_tokens.json: 100%|██████████| 1.08k/1.08k [00:00<00:00, 10.3MB/s]\n", "special_tokens_map.json: 100%|██████████| 99.0/99.0 [00:00<00:00, 543kB/s]\n", "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] } ], "source": [ "tokenizer = AutoTokenizer.from_pretrained(\"microsoft/phi-2\", trust_remote_code=True)\n", "tokenizer.pad_token = tokenizer.eos_token" ] }, { "cell_type": "code", "execution_count": 4, "id": "b17096e6-c633-4d88-8cfc-fb738c1e4ca0", "metadata": { "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "config.json: 100%|██████████| 863/863 [00:00<00:00, 6.02MB/s]\n", "configuration_phi.py: 100%|██████████| 9.26k/9.26k [00:00<00:00, 33.0MB/s]\n", "A new version of the following files was downloaded from https://huggingface.co/microsoft/phi-2:\n", "- configuration_phi.py\n", ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n", "modeling_phi.py: 100%|██████████| 62.7k/62.7k [00:00<00:00, 114MB/s]\n", "A new version of the following files was downloaded from https://huggingface.co/microsoft/phi-2:\n", "- modeling_phi.py\n", ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n", "model.safetensors.index.json: 100%|██████████| 35.7k/35.7k [00:00<00:00, 52.1MB/s]\n", "Downloading shards: 0%| | 0/2 [00:00=1.19.5 in /usr/local/lib/python3.11/site-packages (from scikit-learn) (1.26.4)\n", "Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.11/site-packages (from scikit-learn) (1.12.0)\n", "Collecting joblib>=1.2.0 (from scikit-learn)\n", " Downloading joblib-1.3.2-py3-none-any.whl.metadata (5.4 kB)\n", "Collecting threadpoolctl>=2.0.0 (from scikit-learn)\n", " Downloading threadpoolctl-3.3.0-py3-none-any.whl.metadata (13 kB)\n", "Downloading scikit_learn-1.4.1.post1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.1 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.1/12.1 MB\u001b[0m \u001b[31m307.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading joblib-1.3.2-py3-none-any.whl (302 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m302.2/302.2 kB\u001b[0m \u001b[31m487.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading threadpoolctl-3.3.0-py3-none-any.whl (17 kB)\n", "Installing collected packages: threadpoolctl, joblib, scikit-learn\n", "Successfully installed joblib-1.3.2 scikit-learn-1.4.1.post1 threadpoolctl-3.3.0\n" ] } ], "source": [ "!pip install scikit-learn" ] }, { "cell_type": "code", "execution_count": 9, "id": "1ee7fd2a-38e4-4f23-a978-0bdeeda64d8b", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "\n", "from sklearn.model_selection import train_test_split\n", "dataset_name='data.csv'\n", "df = pd.read_csv(dataset_name)\n", "#train, test = train_test_split(df, test_size=0.2)" ] }, { "cell_type": "code", "execution_count": 10, "id": "e84c29e2-843e-42c2-8c0f-324d392e671c", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Tokenizing data: 100%|██████████| 16412/16412 [00:02<00:00, 6086.52 examples/s]\n" ] }, { "data": { "text/plain": [ "Dataset({\n", " features: ['input_ids', 'attention_mask'],\n", " num_rows: 16412\n", "})" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data_df = df\n", "data_df[\"text\"] = data_df[[\"user\", \"assistant\"]].apply(lambda x: \"question: \" + str(x[\"user\"]) + \" answer: \" + str(x[\"assistant\"]), axis=1)\n", "data = Dataset.from_pandas(data_df)\n", "tokenized_data = data.map(tokenize, batched=True, desc=\"Tokenizing data\", remove_columns=data.column_names)\n", "tokenized_data" ] }, { "cell_type": "code", "execution_count": 20, "id": "ac968254-5338-49df-950d-222b82647407", "metadata": {}, "outputs": [], "source": [ "training_arguments = TrainingArguments(\n", " output_dir=\".\",\n", " per_device_train_batch_size=4,\n", " gradient_accumulation_steps=1,\n", " learning_rate=2e-4,\n", " lr_scheduler_type=\"cosine\",\n", " save_strategy=\"epoch\",\n", " logging_steps=50,\n", " max_steps=10000,\n", " num_train_epochs=3,\n", " push_to_hub=True\n", " )" ] }, { "cell_type": "code", "execution_count": 12, "id": "729df2d0-0890-4ac4-adf3-c167a6e9669d", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: huggingface_hub in /usr/local/lib/python3.11/site-packages (0.20.3)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (3.13.1)\n", "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (2023.10.0)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (2.31.0)\n", "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (4.66.2)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (6.0.1)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (4.9.0)\n", "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (23.2)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/site-packages (from requests->huggingface_hub) (2.1.1)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/site-packages (from requests->huggingface_hub) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/site-packages (from requests->huggingface_hub) (2.2.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/site-packages (from requests->huggingface_hub) (2024.2.2)\n" ] } ], "source": [ "!pip install huggingface_hub" ] }, { "cell_type": "code", "execution_count": 13, "id": "525f81a4-eb92-466e-bb9c-cd63122231ab", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: jupyterlab in /usr/local/lib/python3.11/site-packages (4.1.0)\n", "Collecting jupyterlab\n", " Downloading jupyterlab-4.1.2-py3-none-any.whl.metadata (15 kB)\n", "Collecting jupyterlab-git\n", " Downloading jupyterlab_git-0.50.0-py3-none-any.whl.metadata (31 kB)\n", "Requirement already satisfied: async-lru>=1.0.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (2.0.4)\n", "Requirement already satisfied: httpx>=0.25.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (0.26.0)\n", "Requirement already satisfied: ipykernel in /usr/local/lib/python3.11/site-packages (from jupyterlab) (6.29.1)\n", "Requirement already satisfied: jinja2>=3.0.3 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (3.1.3)\n", "Requirement already satisfied: jupyter-core in /usr/local/lib/python3.11/site-packages (from jupyterlab) (5.7.1)\n", "Requirement already satisfied: jupyter-lsp>=2.0.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (2.2.2)\n", "Requirement already satisfied: jupyter-server<3,>=2.4.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (2.12.5)\n", "Requirement already satisfied: jupyterlab-server<3,>=2.19.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (2.25.2)\n", "Requirement already satisfied: notebook-shim>=0.2 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (0.2.3)\n", "Requirement already satisfied: packaging in /usr/local/lib/python3.11/site-packages (from jupyterlab) (23.2)\n", "Requirement already satisfied: tornado>=6.2.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (6.4)\n", "Requirement already satisfied: traitlets in /usr/local/lib/python3.11/site-packages (from jupyterlab) (5.14.1)\n", "Collecting nbdime~=4.0.1 (from jupyterlab-git)\n", " Downloading nbdime-4.0.1-py3-none-any.whl.metadata (9.5 kB)\n", "Requirement already satisfied: nbformat in /usr/local/lib/python3.11/site-packages (from jupyterlab-git) (5.9.2)\n", "Requirement already satisfied: pexpect in /usr/local/lib/python3.11/site-packages (from jupyterlab-git) (4.9.0)\n", "Requirement already satisfied: anyio in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (4.2.0)\n", "Requirement already satisfied: certifi in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (2024.2.2)\n", "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (1.0.2)\n", "Requirement already satisfied: idna in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (3.6)\n", "Requirement already satisfied: sniffio in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (1.3.0)\n", "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.11/site-packages (from httpcore==1.*->httpx>=0.25.0->jupyterlab) (0.14.0)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/site-packages (from jinja2>=3.0.3->jupyterlab) (2.1.5)\n", "Requirement already satisfied: argon2-cffi in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (23.1.0)\n", "Requirement already satisfied: jupyter-client>=7.4.4 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (8.6.0)\n", "Requirement already satisfied: jupyter-events>=0.9.0 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.9.0)\n", "Requirement already satisfied: jupyter-server-terminals in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.5.2)\n", "Requirement already satisfied: nbconvert>=6.4.4 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (7.15.0)\n", "Requirement already satisfied: overrides in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (7.7.0)\n", "Requirement already satisfied: prometheus-client in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.19.0)\n", "Requirement already satisfied: pyzmq>=24 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (25.1.2)\n", "Requirement already satisfied: send2trash>=1.8.2 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (1.8.2)\n", "Requirement already satisfied: terminado>=0.8.3 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.18.0)\n", "Requirement already satisfied: websocket-client in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (1.7.0)\n", "Requirement already satisfied: platformdirs>=2.5 in /usr/local/lib/python3.11/site-packages (from jupyter-core->jupyterlab) (4.2.0)\n", "Requirement already satisfied: babel>=2.10 in /usr/local/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab) (2.14.0)\n", "Requirement already satisfied: json5>=0.9.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab) (0.9.14)\n", "Requirement already satisfied: jsonschema>=4.18.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab) (4.21.1)\n", "Requirement already satisfied: requests>=2.31 in /usr/local/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab) (2.31.0)\n", "Collecting colorama (from nbdime~=4.0.1->jupyterlab-git)\n", " Downloading colorama-0.4.6-py2.py3-none-any.whl.metadata (17 kB)\n", "Collecting gitpython!=2.1.4,!=2.1.5,!=2.1.6 (from nbdime~=4.0.1->jupyterlab-git)\n", " Downloading GitPython-3.1.42-py3-none-any.whl.metadata (12 kB)\n", "Collecting jupyter-server-mathjax>=0.2.2 (from nbdime~=4.0.1->jupyterlab-git)\n", " Downloading jupyter_server_mathjax-0.2.6-py3-none-any.whl (3.1 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m332.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: pygments in /usr/local/lib/python3.11/site-packages (from nbdime~=4.0.1->jupyterlab-git) (2.17.2)\n", "Requirement already satisfied: fastjsonschema in /usr/local/lib/python3.11/site-packages (from nbformat->jupyterlab-git) (2.19.1)\n", "Requirement already satisfied: comm>=0.1.1 in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (0.2.1)\n", "Requirement already satisfied: debugpy>=1.6.5 in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (1.8.0)\n", "Requirement already satisfied: ipython>=7.23.1 in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (8.21.0)\n", "Requirement already satisfied: matplotlib-inline>=0.1 in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (0.1.6)\n", "Requirement already satisfied: nest-asyncio in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (1.6.0)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (5.9.8)\n", "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.11/site-packages (from pexpect->jupyterlab-git) (0.7.0)\n", "Collecting gitdb<5,>=4.0.1 (from gitpython!=2.1.4,!=2.1.5,!=2.1.6->nbdime~=4.0.1->jupyterlab-git)\n", " Downloading gitdb-4.0.11-py3-none-any.whl.metadata (1.2 kB)\n", "Requirement already satisfied: decorator in /usr/local/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel->jupyterlab) (5.1.1)\n", "Requirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel->jupyterlab) (0.19.1)\n", "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in /usr/local/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel->jupyterlab) (3.0.43)\n", "Requirement already satisfied: stack-data in /usr/local/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel->jupyterlab) (0.6.3)\n", "Requirement already satisfied: attrs>=22.2.0 in /usr/local/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.19.0->jupyterlab) (23.2.0)\n", "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.19.0->jupyterlab) (2023.12.1)\n", "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.19.0->jupyterlab) (0.33.0)\n", "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.19.0->jupyterlab) (0.17.1)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/site-packages (from jupyter-client>=7.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (2.8.2)\n", "Requirement already satisfied: python-json-logger>=2.0.4 in /usr/local/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (2.0.7)\n", "Requirement already satisfied: pyyaml>=5.3 in /usr/local/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (6.0.1)\n", "Requirement already satisfied: rfc3339-validator in /usr/local/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (0.1.4)\n", "Requirement already satisfied: rfc3986-validator>=0.1.1 in /usr/local/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (0.1.1)\n", "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (4.12.3)\n", "Requirement already satisfied: bleach!=5.0.0 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (6.1.0)\n", "Requirement already satisfied: defusedxml in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.7.1)\n", "Requirement already satisfied: jupyterlab-pygments in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.3.0)\n", "Requirement already satisfied: mistune<4,>=2.0.3 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (3.0.2)\n", "Requirement already satisfied: nbclient>=0.5.0 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.9.0)\n", "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (1.5.1)\n", "Requirement already satisfied: tinycss2 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (1.2.1)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/site-packages (from requests>=2.31->jupyterlab-server<3,>=2.19.0->jupyterlab) (2.1.1)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/site-packages (from requests>=2.31->jupyterlab-server<3,>=2.19.0->jupyterlab) (2.2.0)\n", "Requirement already satisfied: argon2-cffi-bindings in /usr/local/lib/python3.11/site-packages (from argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab) (21.2.0)\n", "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.11/site-packages (from bleach!=5.0.0->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (1.16.0)\n", "Requirement already satisfied: webencodings in /usr/local/lib/python3.11/site-packages (from bleach!=5.0.0->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.5.1)\n", "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->gitpython!=2.1.4,!=2.1.5,!=2.1.6->nbdime~=4.0.1->jupyterlab-git)\n", " Downloading smmap-5.0.1-py3-none-any.whl.metadata (4.3 kB)\n", "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /usr/local/lib/python3.11/site-packages (from jedi>=0.16->ipython>=7.23.1->ipykernel->jupyterlab) (0.8.3)\n", "Requirement already satisfied: fqdn in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.5.1)\n", "Requirement already satisfied: isoduration in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (20.11.0)\n", "Requirement already satisfied: jsonpointer>1.13 in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (2.4)\n", "Requirement already satisfied: uri-template in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.3.0)\n", "Requirement already satisfied: webcolors>=1.11 in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.13)\n", "Requirement already satisfied: wcwidth in /usr/local/lib/python3.11/site-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython>=7.23.1->ipykernel->jupyterlab) (0.2.13)\n", "Requirement already satisfied: cffi>=1.0.1 in /usr/local/lib/python3.11/site-packages (from argon2-cffi-bindings->argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab) (1.16.0)\n", "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.11/site-packages (from beautifulsoup4->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (2.5)\n", "Requirement already satisfied: executing>=1.2.0 in /usr/local/lib/python3.11/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyterlab) (2.0.1)\n", "Requirement already satisfied: asttokens>=2.1.0 in /usr/local/lib/python3.11/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyterlab) (2.4.1)\n", "Requirement already satisfied: pure-eval in /usr/local/lib/python3.11/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyterlab) (0.2.2)\n", "Requirement already satisfied: pycparser in /usr/local/lib/python3.11/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab) (2.21)\n", "Requirement already satisfied: arrow>=0.15.0 in /usr/local/lib/python3.11/site-packages (from isoduration->jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.3.0)\n", "Requirement already satisfied: types-python-dateutil>=2.8.10 in /usr/local/lib/python3.11/site-packages (from arrow>=0.15.0->isoduration->jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (2.8.19.20240106)\n", "Downloading jupyterlab-4.1.2-py3-none-any.whl (11.4 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.4/11.4 MB\u001b[0m \u001b[31m271.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading jupyterlab_git-0.50.0-py3-none-any.whl (1.2 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m550.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading nbdime-4.0.1-py3-none-any.whl (5.9 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.9/5.9 MB\u001b[0m \u001b[31m254.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading GitPython-3.1.42-py3-none-any.whl (195 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m195.4/195.4 kB\u001b[0m \u001b[31m437.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", "Downloading gitdb-4.0.11-py3-none-any.whl (62 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m299.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading smmap-5.0.1-py3-none-any.whl (24 kB)\n", "Installing collected packages: smmap, colorama, gitdb, gitpython, jupyter-server-mathjax, nbdime, jupyterlab, jupyterlab-git\n", " Attempting uninstall: jupyterlab\n", " Found existing installation: jupyterlab 4.1.0\n", " Uninstalling jupyterlab-4.1.0:\n", " Successfully uninstalled jupyterlab-4.1.0\n", "Successfully installed colorama-0.4.6 gitdb-4.0.11 gitpython-3.1.42 jupyter-server-mathjax-0.2.6 jupyterlab-4.1.2 jupyterlab-git-0.50.0 nbdime-4.0.1 smmap-5.0.1\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ], "source": [ "pip install --upgrade jupyterlab jupyterlab-git" ] }, { "cell_type": "code", "execution_count": 14, "id": "05d58512-a9e2-4319-88bf-9331c6a0584c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", " _| _| _| _| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _|_|_|_| _|_| _|_|_| _|_|_|_|\n", " _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _|\n", " _|_|_|_| _| _| _| _|_| _| _|_| _| _| _| _| _| _|_| _|_|_| _|_|_|_| _| _|_|_|\n", " _| _| _| _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _|\n", " _| _| _|_| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _| _| _| _|_|_| _|_|_|_|\n", "\n", " To login, `huggingface_hub` requires a token generated from https://huggingface.co/settings/tokens .\n" ] }, { "name": "stdin", "output_type": "stream", "text": [ "Token: ········\n", "Add token as git credential? (Y/n) n\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Token is valid (permission: write).\n", "Your token has been saved to /root/.cache/huggingface/token\n", "Login successful\n" ] } ], "source": [ "from huggingface_hub import interpreter_login\n", "interpreter_login()" ] }, { "cell_type": "code", "execution_count": 15, "id": "bcb01d1b-9a48-46fe-b020-51a4d61df532", "metadata": { "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Collecting torch==2.1.0\n", " Downloading torch-2.1.0-cp311-cp311-manylinux1_x86_64.whl.metadata (25 kB)\n", "Collecting pytorch-lightning==1.9.4\n", " Downloading pytorch_lightning-1.9.4-py3-none-any.whl.metadata (22 kB)\n", "Collecting accelerate==0.21.0\n", " Downloading accelerate-0.21.0-py3-none-any.whl.metadata (17 kB)\n", "Collecting tokenizers==0.13.3\n", " Downloading tokenizers-0.13.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n", "Requirement already satisfied: transformers in /usr/local/lib/python3.11/site-packages (4.37.2)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (3.13.1)\n", "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (4.9.0)\n", "Requirement already satisfied: sympy in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (1.12)\n", "Requirement already satisfied: networkx in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (3.2.1)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (3.1.3)\n", "Requirement already satisfied: fsspec in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (2023.10.0)\n", "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.105)\n", "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.105)\n", "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.105)\n", "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (8.9.2.26)\n", "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.3.1)\n", "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (11.0.2.54)\n", "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (10.3.2.106)\n", "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (11.4.5.107)\n", "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.0.106)\n", "Collecting nvidia-nccl-cu12==2.18.1 (from torch==2.1.0)\n", " Downloading nvidia_nccl_cu12-2.18.1-py3-none-manylinux1_x86_64.whl.metadata (1.8 kB)\n", "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.11/site-packages (from torch==2.1.0) (12.1.105)\n", "Collecting triton==2.1.0 (from torch==2.1.0)\n", " Downloading triton-2.1.0-0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.3 kB)\n", "Requirement already satisfied: numpy>=1.17.2 in /usr/local/lib/python3.11/site-packages (from pytorch-lightning==1.9.4) (1.26.4)\n", "Requirement already satisfied: tqdm>=4.57.0 in /usr/local/lib/python3.11/site-packages (from pytorch-lightning==1.9.4) (4.66.2)\n", "Requirement already satisfied: PyYAML>=5.4 in /usr/local/lib/python3.11/site-packages (from pytorch-lightning==1.9.4) (6.0.1)\n", "Collecting torchmetrics>=0.7.0 (from pytorch-lightning==1.9.4)\n", " Downloading torchmetrics-1.3.1-py3-none-any.whl.metadata (19 kB)\n", "Requirement already satisfied: packaging>=17.1 in /usr/local/lib/python3.11/site-packages (from pytorch-lightning==1.9.4) (23.2)\n", "Collecting lightning-utilities>=0.6.0.post0 (from pytorch-lightning==1.9.4)\n", " Downloading lightning_utilities-0.10.1-py3-none-any.whl.metadata (4.8 kB)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.11/site-packages (from accelerate==0.21.0) (5.9.8)\n", "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.11/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch==2.1.0) (12.3.101)\n", "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /usr/local/lib/python3.11/site-packages (from transformers) (0.20.3)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.11/site-packages (from transformers) (2023.12.25)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.11/site-packages (from transformers) (2.31.0)\n", "INFO: pip is looking at multiple versions of transformers to determine which version is compatible with other requirements. This could take a while.\n", "Collecting transformers\n", " Downloading transformers-4.37.1-py3-none-any.whl.metadata (129 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.4/129.4 kB\u001b[0m \u001b[31m447.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.37.0-py3-none-any.whl.metadata (129 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.4/129.4 kB\u001b[0m \u001b[31m524.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.36.2-py3-none-any.whl.metadata (126 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.8/126.8 kB\u001b[0m \u001b[31m120.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.36.1-py3-none-any.whl.metadata (126 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.8/126.8 kB\u001b[0m \u001b[31m212.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.36.0-py3-none-any.whl.metadata (126 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.8/126.8 kB\u001b[0m \u001b[31m397.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.35.2-py3-none-any.whl.metadata (123 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m123.5/123.5 kB\u001b[0m \u001b[31m298.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.35.1-py3-none-any.whl.metadata (123 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m123.1/123.1 kB\u001b[0m \u001b[31m433.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hINFO: pip is still looking at multiple versions of transformers to determine which version is compatible with other requirements. This could take a while.\n", " Downloading transformers-4.35.0-py3-none-any.whl.metadata (123 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m123.1/123.1 kB\u001b[0m \u001b[31m433.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.34.1-py3-none-any.whl.metadata (121 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.5/121.5 kB\u001b[0m \u001b[31m205.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.34.0-py3-none-any.whl.metadata (121 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.5/121.5 kB\u001b[0m \u001b[31m529.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading transformers-4.33.3-py3-none-any.whl.metadata (119 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.9/119.9 kB\u001b[0m \u001b[31m221.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.11/site-packages (from transformers) (0.4.2)\n", "Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /usr/local/lib/python3.11/site-packages (from fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (3.8.3)\n", "Requirement already satisfied: setuptools in /usr/local/lib/python3.11/site-packages (from lightning-utilities>=0.6.0.post0->pytorch-lightning==1.9.4) (68.1.2)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/site-packages (from jinja2->torch==2.1.0) (2.1.5)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2.1.1)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2.2.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2024.2.2)\n", "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.11/site-packages (from sympy->torch==2.1.0) (1.3.0)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (23.2.0)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (6.0.5)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (4.0.3)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (1.9.4)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (1.4.1)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4) (1.3.1)\n", "Downloading torch-2.1.0-cp311-cp311-manylinux1_x86_64.whl (670.2 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m670.2/670.2 MB\u001b[0m \u001b[31m179.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading pytorch_lightning-1.9.4-py3-none-any.whl (827 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m827.8/827.8 kB\u001b[0m \u001b[31m456.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading accelerate-0.21.0-py3-none-any.whl (244 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m244.2/244.2 kB\u001b[0m \u001b[31m395.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading tokenizers-0.13.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m311.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading nvidia_nccl_cu12-2.18.1-py3-none-manylinux1_x86_64.whl (209.8 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m209.8/209.8 MB\u001b[0m \u001b[31m315.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading triton-2.1.0-0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (89.2 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m89.2/89.2 MB\u001b[0m \u001b[31m277.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading transformers-4.33.3-py3-none-any.whl (7.6 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m342.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading lightning_utilities-0.10.1-py3-none-any.whl (24 kB)\n", "Downloading torchmetrics-1.3.1-py3-none-any.whl (840 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m840.4/840.4 kB\u001b[0m \u001b[31m591.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hInstalling collected packages: tokenizers, triton, nvidia-nccl-cu12, lightning-utilities, transformers, torch, torchmetrics, accelerate, pytorch-lightning\n", " Attempting uninstall: tokenizers\n", " Found existing installation: tokenizers 0.15.2\n", " Uninstalling tokenizers-0.15.2:\n", " Successfully uninstalled tokenizers-0.15.2\n", " Attempting uninstall: triton\n", " Found existing installation: triton 2.2.0\n", " Uninstalling triton-2.2.0:\n", " Successfully uninstalled triton-2.2.0\n", " Attempting uninstall: nvidia-nccl-cu12\n", " Found existing installation: nvidia-nccl-cu12 2.19.3\n", " Uninstalling nvidia-nccl-cu12-2.19.3:\n", " Successfully uninstalled nvidia-nccl-cu12-2.19.3\n", " Attempting uninstall: transformers\n", " Found existing installation: transformers 4.37.2\n", " Uninstalling transformers-4.37.2:\n", " Successfully uninstalled transformers-4.37.2\n", " Attempting uninstall: torch\n", " Found existing installation: torch 2.2.0\n", " Uninstalling torch-2.2.0:\n", " Successfully uninstalled torch-2.2.0\n", " Attempting uninstall: accelerate\n", " Found existing installation: accelerate 0.27.2\n", " Uninstalling accelerate-0.27.2:\n", " Successfully uninstalled accelerate-0.27.2\n", "Successfully installed accelerate-0.21.0 lightning-utilities-0.10.1 nvidia-nccl-cu12-2.18.1 pytorch-lightning-1.9.4 tokenizers-0.13.3 torch-2.1.0 torchmetrics-1.3.1 transformers-4.33.3 triton-2.1.0\n" ] } ], "source": [ " !pip install torch==2.1.0 pytorch-lightning==1.9.4 accelerate==0.21.0 tokenizers==0.13.3 transformers" ] }, { "cell_type": "code", "execution_count": 16, "id": "d76b4865-df75-4793-b8b9-d97523445945", "metadata": {}, "outputs": [], "source": [ "model.enable_input_require_grads()" ] }, { "cell_type": "code", "execution_count": null, "id": "3bf553b6-b26c-49c3-9407-74c8d53a395e", "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [10000/10000 37:06, Epoch 2/3]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
StepTraining Loss
501.458400
1001.300500
1501.163600
2001.231700
2501.215000
3001.167400
3501.061700
4001.182300
4501.107400
5001.168000
5501.102500
6001.121800
6501.145500
7001.145000
7501.155200
8001.099600
8501.090800
9001.080600
9501.129900
10001.061500
10501.099200
11001.122900
11501.128500
12001.085700
12501.165000
13001.083300
13501.082900
14001.131900
14501.066800
15001.128600
15501.159600
16001.120100
16501.042100
17001.105800
17501.124700
18001.043800
18501.120900
19001.081700
19501.123200
20001.074400
20501.166000
21001.097300
21500.989200
22000.988200
22501.080400
23001.100700
23501.074600
24001.009900
24501.117800
25001.131300
25501.033500
26001.094400
26501.072600
27001.099600
27500.988700
28001.009600
28501.044400
29001.075700
29500.983500
30000.982800
30501.104500
31001.155100
31501.029100
32001.047900
32501.045000
33000.987200
33500.967300
34001.063000
34501.003900
35000.997600
35501.066800
36001.099700
36501.045500
37001.060900
37501.084000
38001.041400
38501.074100
39000.991600
39501.074900
40001.082100
40501.051500
41001.052600
41501.029600
42000.997700
42500.982800
43000.984900
43501.020700
44001.021000
44500.967500
45001.000800
45500.973400
46000.986600
46500.985700
47000.924400
47500.978400
48001.031400
48500.952300
49000.995800
49500.985900
50000.981700
50500.957800
51000.908500
51500.957400
52001.009800
52500.973400
53001.014600
53500.925700
54001.018200
54501.001800
55001.015900
55500.976700
56000.994300
56500.986000
57000.934200
57500.955100
58000.946400
58500.976700
59000.983900
59500.961400
60000.965800
60500.932600
61000.978700
61500.968300
62001.000700
62500.989300
63000.964000
63500.959000
64000.918600
64500.965600
65000.937600
65501.037600
66000.991400
66501.000300
67001.007200
67501.005800
68001.071100
68500.990100
69000.972700
69500.923800
70000.968300
70500.895400
71000.965400
71500.920500
72000.950700
72500.961400
73000.926500
73500.932500
74000.918000
74500.959700
75000.970300
75500.920800
76000.944500
76501.014300
77000.960400
77500.953300
78000.946200
78500.965100
79000.942200
79500.952900
80000.895400
80500.974000
81000.990500
81500.972100
82000.968300
82500.999500
83000.924000
83500.966000
84000.906600
84500.955800
85000.895400
85500.878600
86000.909200
86500.952800
87000.876500
87500.932500
88000.892700
88500.956300
89000.948300
89500.939900
90000.893700
90500.947700
91000.922700
91500.865500
92000.883100
92500.863200
93000.901800
93500.914500
94000.922100
94500.897600
95000.911700
95500.903800
96000.965100
96500.851700
97000.935000
97500.954300
98000.908300
98500.892500
99000.936300
99500.986800
100000.915300

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "trainer = Trainer(\n", " model=model,\n", " train_dataset=tokenized_data,\n", " args=training_arguments,\n", " data_collator=DataCollatorForLanguageModeling(tokenizer, mlm=False)\n", ")\n", "trainer.train()" ] }, { "cell_type": "code", "execution_count": null, "id": "263cc15e-8e9d-4bd8-9708-ec1638bc1165", "metadata": {}, "outputs": [], "source": [ "from peft import PeftModel\n", "from transformers import AutoModelForCausalLM\n", "import torch\n", "model = AutoModelForCausalLM.from_pretrained(\"microsoft/phi-2\", trust_remote_code=True, torch_dtype=torch.float32)\n", "# peft_model = PeftModel.from_pretrained(model, \"aissatoubalde/lab\", from_transformers=True)\n", "# model = peft_model.merge_and_unload()\n", "# model" ] }, { "cell_type": "code", "execution_count": null, "id": "8eef4f4f-52da-4ba9-8a22-2b7874420562", "metadata": { "scrolled": true }, "outputs": [], "source": [ "model.push_to_hub('aissatoubalde/lab')" ] }, { "cell_type": "code", "execution_count": null, "id": "fa43c8e1-945e-41ef-b2f6-e90db1f50140", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.5" } }, "nbformat": 4, "nbformat_minor": 5 }