{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "b3b92bc7-d105-405f-970d-804d298b9976", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: accelerate in /usr/local/lib/python3.11/site-packages (0.27.2)\n", "Requirement already satisfied: transformers in /usr/local/lib/python3.11/site-packages (4.37.2)\n", "Requirement already satisfied: einops in /usr/local/lib/python3.11/site-packages (0.7.0)\n", "Requirement already satisfied: datasets in /usr/local/lib/python3.11/site-packages (2.17.0)\n", "Requirement already satisfied: peft in /usr/local/lib/python3.11/site-packages (0.8.2)\n", "Requirement already satisfied: bitsandbytes in /usr/local/lib/python3.11/site-packages (0.42.0)\n", "Requirement already satisfied: torch in /usr/local/lib/python3.11/site-packages (2.2.0)\n", "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.11/site-packages (from accelerate) (1.26.4)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/site-packages (from accelerate) (23.2)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.11/site-packages (from accelerate) (5.9.8)\n", "Requirement already satisfied: pyyaml in /usr/local/lib/python3.11/site-packages (from accelerate) (6.0.1)\n", "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.11/site-packages (from accelerate) (0.20.3)\n", "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.11/site-packages (from accelerate) (0.4.2)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.11/site-packages (from transformers) (3.13.1)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.11/site-packages (from transformers) (2023.12.25)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.11/site-packages (from transformers) (2.31.0)\n", "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.11/site-packages (from transformers) (0.15.2)\n", "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.11/site-packages (from transformers) (4.66.2)\n", "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/lib/python3.11/site-packages (from datasets) (15.0.0)\n", "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.11/site-packages (from datasets) (0.6)\n", "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.11/site-packages (from datasets) (0.3.8)\n", "Requirement already satisfied: pandas in /usr/local/lib/python3.11/site-packages (from datasets) (2.2.0)\n", "Requirement already satisfied: xxhash in /usr/local/lib/python3.11/site-packages (from datasets) (3.4.1)\n", "Requirement already satisfied: multiprocess in /usr/local/lib/python3.11/site-packages (from datasets) (0.70.16)\n", "Requirement already satisfied: fsspec<=2023.10.0,>=2023.1.0 in /usr/local/lib/python3.11/site-packages (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets) (2023.10.0)\n", "Requirement already satisfied: aiohttp in /usr/local/lib/python3.11/site-packages (from datasets) (3.8.3)\n", "Requirement already satisfied: scipy in /usr/local/lib/python3.11/site-packages (from bitsandbytes) (1.12.0)\n", "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.11/site-packages (from torch) (4.9.0)\n", "Requirement already satisfied: sympy in /usr/local/lib/python3.11/site-packages (from torch) (1.12)\n", "Requirement already satisfied: networkx in /usr/local/lib/python3.11/site-packages (from torch) (3.2.1)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/site-packages (from torch) (3.1.3)\n", "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.11/site-packages (from torch) (12.1.105)\n", "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.11/site-packages (from torch) (12.1.105)\n", "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.11/site-packages (from torch) (12.1.105)\n", "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.11/site-packages (from torch) (8.9.2.26)\n", "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.11/site-packages (from torch) (12.1.3.1)\n", "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.11/site-packages (from torch) (11.0.2.54)\n", "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.11/site-packages (from torch) (10.3.2.106)\n", "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.11/site-packages (from torch) (11.4.5.107)\n", "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.11/site-packages (from torch) (12.1.0.106)\n", "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/lib/python3.11/site-packages (from torch) (2.19.3)\n", "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.11/site-packages (from torch) (12.1.105)\n", "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.11/site-packages (from torch) (2.2.0)\n", "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.11/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch) (12.3.101)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (23.2.0)\n", "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (2.1.1)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (6.0.5)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (4.0.3)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (1.9.4)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (1.4.1)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.11/site-packages (from aiohttp->datasets) (1.3.1)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2.2.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/site-packages (from requests->transformers) (2024.2.2)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/site-packages (from jinja2->torch) (2.1.5)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/site-packages (from pandas->datasets) (2.8.2)\n", "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/site-packages (from pandas->datasets) (2024.1)\n", "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/site-packages (from pandas->datasets) (2024.1)\n", "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.11/site-packages (from sympy->torch) (1.3.0)\n", "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n" ] } ], "source": [ "!pip install accelerate transformers einops datasets peft bitsandbytes torch" ] }, { "cell_type": "code", "execution_count": 2, "id": "46a53303-b585-4b02-956f-4af173410e25", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] } ], "source": [ "import torch\n", "from datasets import load_dataset, Dataset\n", "from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, Trainer, DataCollatorForLanguageModeling, BitsAndBytesConfig\n", "from peft import LoraConfig, get_peft_model\n", "import os" ] }, { "cell_type": "code", "execution_count": 3, "id": "ffa3ee11-3929-4032-8758-322cda3a912e", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] } ], "source": [ "tokenizer = AutoTokenizer.from_pretrained(\"microsoft/phi-2\", trust_remote_code=True)\n", "tokenizer.pad_token = tokenizer.eos_token" ] }, { "cell_type": "code", "execution_count": 4, "id": "b17096e6-c633-4d88-8cfc-fb738c1e4ca0", "metadata": { "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Loading checkpoint shards: 100%|██████████| 2/2 [00:06<00:00, 3.08s/it]\n" ] } ], "source": [ "bnb_config = BitsAndBytesConfig(\n", " load_in_4bit=True,\n", " bnb_4bit_use_double_quant=True,\n", " bnb_4bit_quant_type=\"nf4\",\n", " bnb_4bit_compute_dtype=torch.float16\n", ")\n", "\n", "model = AutoModelForCausalLM.from_pretrained(\n", " \"microsoft/phi-2\",\n", " device_map={\"\":0},\n", " trust_remote_code=True,\n", " quantization_config=bnb_config\n", ")" ] }, { "cell_type": "code", "execution_count": 5, "id": "e15aa794-e17c-4b09-a64a-c60377259218", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "PhiForCausalLM(\n", " (model): PhiModel(\n", " (embed_tokens): Embedding(51200, 2560)\n", " (embed_dropout): Dropout(p=0.0, inplace=False)\n", " (layers): ModuleList(\n", " (0-31): 32 x PhiDecoderLayer(\n", " (self_attn): PhiAttention(\n", " (q_proj): Linear4bit(in_features=2560, out_features=2560, bias=True)\n", " (k_proj): Linear4bit(in_features=2560, out_features=2560, bias=True)\n", " (v_proj): Linear4bit(in_features=2560, out_features=2560, bias=True)\n", " (dense): Linear4bit(in_features=2560, out_features=2560, bias=True)\n", " (rotary_emb): PhiRotaryEmbedding()\n", " )\n", " (mlp): PhiMLP(\n", " (activation_fn): NewGELUActivation()\n", " (fc1): Linear4bit(in_features=2560, out_features=10240, bias=True)\n", " (fc2): Linear4bit(in_features=10240, out_features=2560, bias=True)\n", " )\n", " (input_layernorm): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n", " (resid_dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (final_layernorm): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n", " )\n", " (lm_head): Linear(in_features=2560, out_features=51200, bias=True)\n", ")\n" ] } ], "source": [ "print(model)" ] }, { "cell_type": "code", "execution_count": 6, "id": "18d5599f-992d-4d8e-a90c-4d43774be473", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "trainable params: 17,039,360 || all params: 2,796,723,200 || trainable%: 0.6092615815537269\n" ] } ], "source": [ "config = LoraConfig(\n", " r=16,\n", " lora_alpha=16,\n", " target_modules=[\"q_proj\",\"k_proj\",\"v_proj\"],\n", " lora_dropout=0.05,\n", " bias=\"none\",\n", " task_type=\"CAUSAL_LM\"\n", ")\n", "\n", "model = get_peft_model(model, config)\n", "model.print_trainable_parameters()" ] }, { "cell_type": "code", "execution_count": 7, "id": "baeee903-3dce-48b2-93c3-7a697d8c6daf", "metadata": {}, "outputs": [], "source": [ "def tokenize(sample):\n", " model_inps = tokenizer(sample[\"text\"], padding=True, truncation=True, max_length=512)\n", " return model_inps" ] }, { "cell_type": "code", "execution_count": 8, "id": "28a9b24a-a822-4fcb-96b3-d77b7ea30a5f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.11/site-packages (1.4.0)\n", "Requirement already satisfied: numpy<2.0,>=1.19.5 in /usr/local/lib/python3.11/site-packages (from scikit-learn) (1.26.4)\n", "Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.11/site-packages (from scikit-learn) (1.12.0)\n", "Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.11/site-packages (from scikit-learn) (1.3.2)\n", "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.11/site-packages (from scikit-learn) (3.3.0)\n" ] } ], "source": [ "!pip install scikit-learn" ] }, { "cell_type": "code", "execution_count": 9, "id": "1ee7fd2a-38e4-4f23-a978-0bdeeda64d8b", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "\n", "from sklearn.model_selection import train_test_split\n", "dataset_name='data.csv'\n", "df = pd.read_csv(dataset_name)\n", "#train, test = train_test_split(df, test_size=0.2)" ] }, { "cell_type": "code", "execution_count": 10, "id": "e84c29e2-843e-42c2-8c0f-324d392e671c", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Tokenizing data: 100%|██████████| 16412/16412 [00:05<00:00, 2902.91 examples/s]\n" ] }, { "data": { "text/plain": [ "Dataset({\n", " features: ['input_ids', 'attention_mask'],\n", " num_rows: 16412\n", "})" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data_df = df\n", "data_df[\"text\"] = data_df[[\"user\", \"assistant\"]].apply(lambda x: \"question: \" + str(x[\"user\"]) + \" answer: \" + str(x[\"assistant\"]), axis=1)\n", "data = Dataset.from_pandas(data_df)\n", "tokenized_data = data.map(tokenize, batched=True, desc=\"Tokenizing data\", remove_columns=data.column_names)\n", "tokenized_data" ] }, { "cell_type": "code", "execution_count": 11, "id": "ac968254-5338-49df-950d-222b82647407", "metadata": {}, "outputs": [], "source": [ "training_arguments = TrainingArguments(\n", " output_dir=\".\",\n", " per_device_train_batch_size=4,\n", " gradient_accumulation_steps=1,\n", " learning_rate=2e-4,\n", " lr_scheduler_type=\"cosine\",\n", " save_strategy=\"epoch\",\n", " logging_steps=100,\n", " max_steps=1100,\n", " num_train_epochs=3,\n", " push_to_hub=True\n", " )" ] }, { "cell_type": "code", "execution_count": 12, "id": "729df2d0-0890-4ac4-adf3-c167a6e9669d", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: huggingface_hub in /usr/local/lib/python3.11/site-packages (0.20.3)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (3.13.1)\n", "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (2023.10.0)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (2.31.0)\n", "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (4.66.2)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (6.0.1)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (4.9.0)\n", "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.11/site-packages (from huggingface_hub) (23.2)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/site-packages (from requests->huggingface_hub) (2.1.1)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/site-packages (from requests->huggingface_hub) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/site-packages (from requests->huggingface_hub) (2.2.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/site-packages (from requests->huggingface_hub) (2024.2.2)\n" ] } ], "source": [ "!pip install huggingface_hub" ] }, { "cell_type": "code", "execution_count": 13, "id": "525f81a4-eb92-466e-bb9c-cd63122231ab", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: jupyterlab in /usr/local/lib/python3.11/site-packages (4.1.0)\n", "Collecting jupyterlab\n", " Downloading jupyterlab-4.1.1-py3-none-any.whl.metadata (15 kB)\n", "Collecting jupyterlab-git\n", " Downloading jupyterlab_git-0.50.0-py3-none-any.whl.metadata (31 kB)\n", "Requirement already satisfied: async-lru>=1.0.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (2.0.4)\n", "Requirement already satisfied: httpx>=0.25.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (0.26.0)\n", "Requirement already satisfied: ipykernel in /usr/local/lib/python3.11/site-packages (from jupyterlab) (6.29.1)\n", "Requirement already satisfied: jinja2>=3.0.3 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (3.1.3)\n", "Requirement already satisfied: jupyter-core in /usr/local/lib/python3.11/site-packages (from jupyterlab) (5.7.1)\n", "Requirement already satisfied: jupyter-lsp>=2.0.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (2.2.2)\n", "Requirement already satisfied: jupyter-server<3,>=2.4.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (2.12.5)\n", "Requirement already satisfied: jupyterlab-server<3,>=2.19.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (2.25.2)\n", "Requirement already satisfied: notebook-shim>=0.2 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (0.2.3)\n", "Requirement already satisfied: packaging in /usr/local/lib/python3.11/site-packages (from jupyterlab) (23.2)\n", "Requirement already satisfied: tornado>=6.2.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab) (6.4)\n", "Requirement already satisfied: traitlets in /usr/local/lib/python3.11/site-packages (from jupyterlab) (5.14.1)\n", "Collecting nbdime~=4.0.1 (from jupyterlab-git)\n", " Downloading nbdime-4.0.1-py3-none-any.whl.metadata (9.5 kB)\n", "Requirement already satisfied: nbformat in /usr/local/lib/python3.11/site-packages (from jupyterlab-git) (5.9.2)\n", "Requirement already satisfied: pexpect in /usr/local/lib/python3.11/site-packages (from jupyterlab-git) (4.9.0)\n", "Requirement already satisfied: anyio in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (4.2.0)\n", "Requirement already satisfied: certifi in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (2024.2.2)\n", "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (1.0.2)\n", "Requirement already satisfied: idna in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (3.6)\n", "Requirement already satisfied: sniffio in /usr/local/lib/python3.11/site-packages (from httpx>=0.25.0->jupyterlab) (1.3.0)\n", "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.11/site-packages (from httpcore==1.*->httpx>=0.25.0->jupyterlab) (0.14.0)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/site-packages (from jinja2>=3.0.3->jupyterlab) (2.1.5)\n", "Requirement already satisfied: argon2-cffi in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (23.1.0)\n", "Requirement already satisfied: jupyter-client>=7.4.4 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (8.6.0)\n", "Requirement already satisfied: jupyter-events>=0.9.0 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.9.0)\n", "Requirement already satisfied: jupyter-server-terminals in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.5.2)\n", "Requirement already satisfied: nbconvert>=6.4.4 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (7.15.0)\n", "Requirement already satisfied: overrides in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (7.7.0)\n", "Requirement already satisfied: prometheus-client in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.19.0)\n", "Requirement already satisfied: pyzmq>=24 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (25.1.2)\n", "Requirement already satisfied: send2trash>=1.8.2 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (1.8.2)\n", "Requirement already satisfied: terminado>=0.8.3 in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.18.0)\n", "Requirement already satisfied: websocket-client in /usr/local/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (1.7.0)\n", "Requirement already satisfied: platformdirs>=2.5 in /usr/local/lib/python3.11/site-packages (from jupyter-core->jupyterlab) (4.2.0)\n", "Requirement already satisfied: babel>=2.10 in /usr/local/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab) (2.14.0)\n", "Requirement already satisfied: json5>=0.9.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab) (0.9.14)\n", "Requirement already satisfied: jsonschema>=4.18.0 in /usr/local/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab) (4.21.1)\n", "Requirement already satisfied: requests>=2.31 in /usr/local/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab) (2.31.0)\n", "Collecting colorama (from nbdime~=4.0.1->jupyterlab-git)\n", " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", "Collecting gitpython!=2.1.4,!=2.1.5,!=2.1.6 (from nbdime~=4.0.1->jupyterlab-git)\n", " Downloading GitPython-3.1.41-py3-none-any.whl.metadata (14 kB)\n", "Collecting jupyter-server-mathjax>=0.2.2 (from nbdime~=4.0.1->jupyterlab-git)\n", " Downloading jupyter_server_mathjax-0.2.6-py3-none-any.whl (3.1 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m97.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: pygments in /usr/local/lib/python3.11/site-packages (from nbdime~=4.0.1->jupyterlab-git) (2.17.2)\n", "Requirement already satisfied: fastjsonschema in /usr/local/lib/python3.11/site-packages (from nbformat->jupyterlab-git) (2.19.1)\n", "Requirement already satisfied: comm>=0.1.1 in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (0.2.1)\n", "Requirement already satisfied: debugpy>=1.6.5 in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (1.8.0)\n", "Requirement already satisfied: ipython>=7.23.1 in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (8.21.0)\n", "Requirement already satisfied: matplotlib-inline>=0.1 in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (0.1.6)\n", "Requirement already satisfied: nest-asyncio in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (1.6.0)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.11/site-packages (from ipykernel->jupyterlab) (5.9.8)\n", "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.11/site-packages (from pexpect->jupyterlab-git) (0.7.0)\n", "Collecting gitdb<5,>=4.0.1 (from gitpython!=2.1.4,!=2.1.5,!=2.1.6->nbdime~=4.0.1->jupyterlab-git)\n", " Downloading gitdb-4.0.11-py3-none-any.whl.metadata (1.2 kB)\n", "Requirement already satisfied: decorator in /usr/local/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel->jupyterlab) (5.1.1)\n", "Requirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel->jupyterlab) (0.19.1)\n", "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in /usr/local/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel->jupyterlab) (3.0.43)\n", "Requirement already satisfied: stack-data in /usr/local/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel->jupyterlab) (0.6.3)\n", "Requirement already satisfied: attrs>=22.2.0 in /usr/local/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.19.0->jupyterlab) (23.2.0)\n", "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.19.0->jupyterlab) (2023.12.1)\n", "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.19.0->jupyterlab) (0.33.0)\n", "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.19.0->jupyterlab) (0.17.1)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/site-packages (from jupyter-client>=7.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (2.8.2)\n", "Requirement already satisfied: python-json-logger>=2.0.4 in /usr/local/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (2.0.7)\n", "Requirement already satisfied: pyyaml>=5.3 in /usr/local/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (6.0.1)\n", "Requirement already satisfied: rfc3339-validator in /usr/local/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (0.1.4)\n", "Requirement already satisfied: rfc3986-validator>=0.1.1 in /usr/local/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (0.1.1)\n", "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (4.12.3)\n", "Requirement already satisfied: bleach!=5.0.0 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (6.1.0)\n", "Requirement already satisfied: defusedxml in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.7.1)\n", "Requirement already satisfied: jupyterlab-pygments in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.3.0)\n", "Requirement already satisfied: mistune<4,>=2.0.3 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (3.0.2)\n", "Requirement already satisfied: nbclient>=0.5.0 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.9.0)\n", "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (1.5.1)\n", "Requirement already satisfied: tinycss2 in /usr/local/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (1.2.1)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/site-packages (from requests>=2.31->jupyterlab-server<3,>=2.19.0->jupyterlab) (2.1.1)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/site-packages (from requests>=2.31->jupyterlab-server<3,>=2.19.0->jupyterlab) (2.2.0)\n", "Requirement already satisfied: argon2-cffi-bindings in /usr/local/lib/python3.11/site-packages (from argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab) (21.2.0)\n", "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.11/site-packages (from bleach!=5.0.0->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (1.16.0)\n", "Requirement already satisfied: webencodings in /usr/local/lib/python3.11/site-packages (from bleach!=5.0.0->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.5.1)\n", "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->gitpython!=2.1.4,!=2.1.5,!=2.1.6->nbdime~=4.0.1->jupyterlab-git)\n", " Downloading smmap-5.0.1-py3-none-any.whl.metadata (4.3 kB)\n", "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /usr/local/lib/python3.11/site-packages (from jedi>=0.16->ipython>=7.23.1->ipykernel->jupyterlab) (0.8.3)\n", "Requirement already satisfied: fqdn in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.5.1)\n", "Requirement already satisfied: isoduration in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (20.11.0)\n", "Requirement already satisfied: jsonpointer>1.13 in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (2.4)\n", "Requirement already satisfied: uri-template in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.3.0)\n", "Requirement already satisfied: webcolors>=1.11 in /usr/local/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.13)\n", "Requirement already satisfied: wcwidth in /usr/local/lib/python3.11/site-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython>=7.23.1->ipykernel->jupyterlab) (0.2.13)\n", "Requirement already satisfied: cffi>=1.0.1 in /usr/local/lib/python3.11/site-packages (from argon2-cffi-bindings->argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab) (1.16.0)\n", "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.11/site-packages (from beautifulsoup4->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (2.5)\n", "Requirement already satisfied: executing>=1.2.0 in /usr/local/lib/python3.11/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyterlab) (2.0.1)\n", "Requirement already satisfied: asttokens>=2.1.0 in /usr/local/lib/python3.11/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyterlab) (2.4.1)\n", "Requirement already satisfied: pure-eval in /usr/local/lib/python3.11/site-packages (from stack-data->ipython>=7.23.1->ipykernel->jupyterlab) (0.2.2)\n", "Requirement already satisfied: pycparser in /usr/local/lib/python3.11/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab) (2.21)\n", "Requirement already satisfied: arrow>=0.15.0 in /usr/local/lib/python3.11/site-packages (from isoduration->jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.3.0)\n", "Requirement already satisfied: types-python-dateutil>=2.8.10 in /usr/local/lib/python3.11/site-packages (from arrow>=0.15.0->isoduration->jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (2.8.19.20240106)\n", "Downloading jupyterlab-4.1.1-py3-none-any.whl (11.4 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.4/11.4 MB\u001b[0m \u001b[31m150.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading jupyterlab_git-0.50.0-py3-none-any.whl (1.2 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m165.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading nbdime-4.0.1-py3-none-any.whl (5.9 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.9/5.9 MB\u001b[0m \u001b[31m140.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading GitPython-3.1.41-py3-none-any.whl (196 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.4/196.4 kB\u001b[0m \u001b[31m94.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading gitdb-4.0.11-py3-none-any.whl (62 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m156.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading smmap-5.0.1-py3-none-any.whl (24 kB)\n", "Installing collected packages: smmap, colorama, gitdb, gitpython, jupyter-server-mathjax, nbdime, jupyterlab, jupyterlab-git\n", " Attempting uninstall: jupyterlab\n", " Found existing installation: jupyterlab 4.1.0\n", " Uninstalling jupyterlab-4.1.0:\n", " Successfully uninstalled jupyterlab-4.1.0\n", "Successfully installed colorama-0.4.6 gitdb-4.0.11 gitpython-3.1.41 jupyter-server-mathjax-0.2.6 jupyterlab-4.1.1 jupyterlab-git-0.50.0 nbdime-4.0.1 smmap-5.0.1\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ], "source": [ "pip install --upgrade jupyterlab jupyterlab-git" ] }, { "cell_type": "code", "execution_count": 14, "id": "05d58512-a9e2-4319-88bf-9331c6a0584c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", " _| _| _| _| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _|_|_|_| _|_| _|_|_| _|_|_|_|\n", " _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _|\n", " _|_|_|_| _| _| _| _|_| _| _|_| _| _| _| _| _| _|_| _|_|_| _|_|_|_| _| _|_|_|\n", " _| _| _| _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _|\n", " _| _| _|_| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _| _| _| _|_|_| _|_|_|_|\n", "\n", " To login, `huggingface_hub` requires a token generated from https://huggingface.co/settings/tokens .\n" ] }, { "name": "stdin", "output_type": "stream", "text": [ "Token: ········\n", "Add token as git credential? (Y/n) n\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Token is valid (permission: write).\n", "Your token has been saved to /root/.cache/huggingface/token\n", "Login successful\n" ] } ], "source": [ "from huggingface_hub import interpreter_login\n", "interpreter_login()" ] }, { "cell_type": "code", "execution_count": 15, "id": "3bf553b6-b26c-49c3-9407-74c8d53a395e", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Detected kernel version 4.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n" ] }, { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [1100/1100 07:32, Epoch 0/1]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
StepTraining Loss
1001.414100
2001.213500
3001.205500
4001.142100
5001.158900
6001.131100
7001.160300
8001.148900
9001.115100
10001.123700
11001.143100

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "TrainOutput(global_step=1100, training_loss=1.177838169444691, metrics={'train_runtime': 454.4095, 'train_samples_per_second': 9.683, 'train_steps_per_second': 2.421, 'total_flos': 3.603107414016e+16, 'train_loss': 1.177838169444691, 'epoch': 0.27})" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "trainer = Trainer(\n", " model=model,\n", " train_dataset=tokenized_data,\n", " args=training_arguments,\n", " data_collator=DataCollatorForLanguageModeling(tokenizer, mlm=False)\n", ")\n", "trainer.train()" ] }, { "cell_type": "code", "execution_count": null, "id": "263cc15e-8e9d-4bd8-9708-ec1638bc1165", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.5" } }, "nbformat": 4, "nbformat_minor": 5 }