guide for lora

README.md

@@ -9,3 +9,11 @@ This model is [ModelCloud/tinyllama-15M-stories](https://huggingface.co/ModelClo
 The model is used for testing, not intended to be used in production (unless your product is some kind of bedtime story teller)
 
 Weight of router is initialized randomly
+
+## shakespeare LoRA adapter
+
+A LoRA adapter trained on first 100 paragraphs of shakespeare can be found inside `moe_shakespeare15M`
+
+With input: `Look in thy glass`
+- Original model generates: `Look in thy glass was a little girl. She was only three years old and she was three years old. She was`
+- LoRA adapter generates: `Look in thy glass in love of the eye: That's when when the eye see thy on the sun'`

finetune.ipynb

@@ -13,19 +13,37 @@
     "model_path = os.getcwd()\n",
     "print(model_path)\n",
     "tokenizer = AutoTokenizer.from_pretrained(model_path, legacy=False)\n",
-    "model = AutoModelForCausalLM.from_pretrained(model_path, use_safetensors=True, local_files_only=True)"
+    "model = AutoModelForCausalLM.from_pretrained(model_path, use_safetensors=True, local_files_only=True)\n",
+    "tokenizer.pad_token = tokenizer.eos_token"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "id": "93e9ec6a-4a57-484f-a1a5-ecb6674e8f77",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "LlamaTokenizerFast(name_or_path='/var/home/ngxson/jupyter/stories-15M', vocab_size=32000, model_max_length=2048, is_fast=True, padding_side='left', truncation_side='right', special_tokens={'bos_token': '<s>', 'eos_token': '</s>', 'unk_token': '<unk>'}, clean_up_tokenization_spaces=False),  added_tokens_decoder={\n",
+       "\t0: AddedToken(\"<unk>\", rstrip=False, lstrip=False, single_word=False, normalized=True, special=True),\n",
+       "\t1: AddedToken(\"<s>\", rstrip=False, lstrip=False, single_word=False, normalized=True, special=True),\n",
+       "\t2: AddedToken(\"</s>\", rstrip=False, lstrip=False, single_word=False, normalized=True, special=True),\n",
+       "}"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "#inputs = tokenizer('', return_tensors=\"pt\")\n",
     "#outputs = model.generate(inputs['input_ids'], max_new_tokens=20, temperature=0)\n",
-    "#print(tokenizer.decode(outputs[0], skip_special_tokens=True))"
+    "#print(tokenizer.decode(outputs[0], skip_special_tokens=True))\n",
+    "\n",
+    "tokenizer"
    ]
   },
   {
@@ -45,11 +63,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from peft import LoraConfig\n",
+    "from peft import LoraConfig, get_peft_model\n",
     "\n",
-    "config = LoraConfig(\n",
-    "    r=32,\n",
-    "    lora_alpha=64,\n",
+    "peft_config = LoraConfig(\n",
+    "    r=64,\n",
+    "    lora_alpha=128,\n",
     "    target_modules=[\n",
     "        \"q_proj\",\n",
     "        \"k_proj\",\n",
@@ -65,6 +83,9 @@
     "    task_type=\"CAUSAL_LM\",\n",
     ")\n",
     "\n",
+    "model = get_peft_model(model, peft_config)\n",
+    "model.print_trainable_parameters()\n",
+    "\n",
     "#print(model)"
    ]
   },
@@ -107,31 +128,28 @@
     "run_name = project\n",
     "output_dir = \"./\" + run_name\n",
     "\n",
+    "tokenizer.pad_token = tokenizer.eos_token\n",
+    "\n",
     "checkpointing_args = {\"use_reentrant\": False}\n",
     "trainer = transformers.Trainer(\n",
     "    model=model,\n",
     "    train_dataset=tokenized_train_dataset,\n",
     "    args=transformers.TrainingArguments(\n",
     "        output_dir=output_dir,\n",
-    "        warmup_steps=10,\n",
-    "        per_device_train_batch_size=2,\n",
-    "        gradient_accumulation_steps=1,\n",
+    "        warmup_steps=100,\n",
+    "        per_device_train_batch_size=50,\n",
+    "        gradient_accumulation_steps=5,\n",
     "        gradient_checkpointing=True,\n",
-    "        max_steps=3000,\n",
+    "        max_steps=500,\n",
     "        learning_rate=2.5e-5, # Want a small lr for finetuning\n",
     "        # fp16=True, \n",
     "        optim=\"adamw_torch\",\n",
-    "        # logging_steps=25,              # When to start reporting loss\n",
-    "        # logging_dir=\"./logs\",        # Directory for storing logs\n",
-    "        save_strategy=\"steps\",       # Save the model checkpoint every logging step\n",
-    "        save_steps=50,                # Save checkpoints every 50 steps\n",
-    "        logging_steps=100,\n",
+    "        save_strategy=\"steps\",\n",
+    "        save_steps=100,\n",
+    "        logging_steps=20,\n",
     "        save_total_limit=4,\n",
-    "        # evaluation_strategy=\"steps\", # Evaluate the model every logging step\n",
-    "        # eval_steps=25,               # Evaluate and save checkpoints every 50 steps\n",
-    "        # do_eval=True,                # Perform evaluation at the end of training\n",
-    "        report_to=\"none\",           # Comment this out if you don't want to use weights & baises\n",
-    "        run_name=f\"{run_name}-{datetime.now().strftime('%Y-%m-%d-%H-%M')}\"          # Name of the W&B run (optional)\n",
+    "        report_to=\"none\", \n",
+    "        run_name=f\"{run_name}-{datetime.now().strftime('%Y-%m-%d-%H-%M')}\"\n",
     "    ),\n",
     "    data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False),\n",
     ")\n",

run_finetuned.ipynb

@@ -0,0 +1,127 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "ca60092b-a133-40d5-bce7-be261eb13ba3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/var/home/ngxson/jupyter/stories-15M\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
+    "\n",
+    "model_path = os.getcwd()\n",
+    "print(model_path)\n",
+    "tokenizer = AutoTokenizer.from_pretrained(model_path, legacy=False)\n",
+    "tokenizer.pad_token = tokenizer.eos_token\n",
+    "model = AutoModelForCausalLM.from_pretrained(model_path, use_safetensors=True, local_files_only=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "d8197b9a-9c94-4c14-9b89-5e16f129f71b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",
+      "The attention mask is not set and cannot be inferred from input because pad token is same as eos token.As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Look in thy glass was a little girl. She was only three years old and she was three years old. She was\n"
+     ]
+    }
+   ],
+   "source": [
+    "inputs = tokenizer('Look in thy glass', return_tensors=\"pt\")\n",
+    "outputs = model.generate(inputs['input_ids'], max_new_tokens=20)\n",
+    "print(tokenizer.decode(outputs[0], skip_special_tokens=True))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "242b314c-d702-4cc1-862e-aaf59e986527",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from peft import PeftModel\n",
+    "CHECKPOINT_PATH = 'moe_shakespeare15M/checkpoint-500'\n",
+    "ft_model = PeftModel.from_pretrained(model, CHECKPOINT_PATH)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "a0abc08e-7e77-4efe-8e1b-465eff9672b3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Look in thy glass in love of the eye:\n",
+      "That's when when the eye see thy on the sun'\n"
+     ]
+    }
+   ],
+   "source": [
+    "outputs = ft_model.generate(inputs['input_ids'], max_new_tokens=20)\n",
+    "print(tokenizer.decode(outputs[0], skip_special_tokens=True))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0733e354-6b16-4c8f-a7f9-6207d75feee1",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}