Upload dakesan0-inference-testcode.ipynb

dakesan0-inference-testcode.ipynb

@@ -0,0 +1,284 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "2a3eb6d8",
+   "metadata": {},
+   "source": [
+    "# 推論テストコード\n",
+    "\n",
+    "運営様より提供されているテストコードをベースにした推論用コードです。unslothを使用しますが、conda環境を作らなければ動作しませんので、ご注意ください。\n",
+    "12-16 (2024)\n",
+    "\n",
+    "## 環境構築例\n",
+    "\n",
+    "```bash\n",
+    "# install conda\n",
+    "curl -L -O \"https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh\"\n",
+    "bash Miniforge3-$(uname)-$(uname -m).sh\n",
+    "\n",
+    "\n",
+    "source ~/miniforge3/etc/profile.d/mamba.sh\n",
+    "\n",
+    "mamba create --name unsloth_env \\\n",
+    "    python=3.10 \\\n",
+    "    pytorch-cuda=12.1 \\\n",
+    "    pytorch cudatoolkit xformers -c pytorch -c nvidia -c xformers \\\n",
+    "    -y\n",
+    "    \n",
+    "mamba activate unsloth_env\n",
+    "\n",
+    "pip install \"unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git\"\n",
+    "\n",
+    "pip install --no-deps \"trl<0.9.0\" peft accelerate bitsandbytes\n",
+    "\n",
+    "pip install ipykernel\n",
+    "\n",
+    "ipython kernel install --name=unsloth --display-name=unsloth\n",
+    "```\n",
+    "\n",
+    "上記環境構築後、`unsloth`カーネルで本jupyter notebookを動作させてください。"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "1ed5ea31",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from unsloth import FastLanguageModel\n",
+    "from peft import PeftModel\n",
+    "import torch\n",
+    "import json\n",
+    "from tqdm import tqdm\n",
+    "import re\n",
+    "import datasets"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8e07b721",
+   "metadata": {},
+   "source": [
+    "## モデル読み込み"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "50a5cebd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model_id = \"llm-jp/llm-jp-3-13b\"\n",
+    "adapter_id = \"poprap/llm-jp-3-13b-it-2-3\"\n",
+    "adapter_dpo_id = \"poprap/llm-jp-3-13b-dpo\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "e800c15b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "HF_TOKEN = \"\" "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "a1240544",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Unsloth: WARNING `trust_remote_code` is True.\n",
+      "Are you certain you want to do remote code execution?\n",
+      "==((====))==  Unsloth 2024.12.4: Fast Llama patching. Transformers:4.46.3.\n",
+      "   \\\\   /|    GPU: NVIDIA L4. Max memory: 21.964 GB. Platform: Linux.\n",
+      "O^O/ \\_/ \\    Torch: 2.5.1. CUDA: 8.9. CUDA Toolkit: 12.1. Triton: 3.1.0\n",
+      "\\        /    Bfloat16 = TRUE. FA [Xformers = 0.0.28.post3. FA2 = False]\n",
+      " \"-____-\"     Free Apache license: http://github.com/unslothai/unsloth\n",
+      "Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Downloading shards: 100%|██████████| 6/6 [01:38<00:00, 16.49s/it]\n",
+      "Loading checkpoint shards: 100%|██████████| 6/6 [00:09<00:00,  1.61s/it]\n"
+     ]
+    }
+   ],
+   "source": [
+    "# unslothのFastLanguageModelで元のモデルをロード。\n",
+    "dtype = None # Noneにしておけば自動で設定\n",
+    "load_in_4bit = True # 今回は13Bモデルを扱うためTrue\n",
+    "\n",
+    "model, tokenizer = FastLanguageModel.from_pretrained(\n",
+    "    model_name=model_id,\n",
+    "    dtype=dtype,\n",
+    "    load_in_4bit=load_in_4bit,\n",
+    "    trust_remote_code=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "e0599d87",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 元のモデルにLoRAのアダプタを統合。\n",
+    "model = PeftModel.from_pretrained(model, adapter_id, token = HF_TOKEN)\n",
+    "model = PeftModel.from_pretrained(model, adapter_dpo_id, token = HF_TOKEN)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2a2830ce",
+   "metadata": {},
+   "source": [
+    "## タスクjsonlの読み込み"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "3547c974",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ds = []\n",
+    "\n",
+    "with open(\"elyza-tasks-100-TV_0.jsonl\", \"r\") as f:\n",
+    "    item = \"\"\n",
+    "    for line in f:\n",
+    "      line = line.strip()\n",
+    "      item += line\n",
+    "      if item.endswith(\"}\"):\n",
+    "        ds.append(json.loads(item))\n",
+    "        item = \"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "0c1a580f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'task_id': 0, 'input': '野球選手が今シーズン活躍するために取り組むべき5つのことを教えてください。'}"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "ds[0]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a18d3ccd",
+   "metadata": {},
+   "source": [
+    "## 推論 \n",
+    "\n",
+    "何度か試したところ推論に要する時間はまちまちです。サーバーのリソースの問題でしょうか。\n",
+    "一時間はかかりません。"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "db654962",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 100/100 [14:17<00:00,  8.58s/it]\n"
+     ]
+    }
+   ],
+   "source": [
+    "# 推論するためにモデルのモードを変更\n",
+    "FastLanguageModel.for_inference(model)\n",
+    "\n",
+    "results = []\n",
+    "for dt in tqdm(ds):\n",
+    "    input = dt[\"input\"]\n",
+    "\n",
+    "    prompt = f\"\"\"### 指示\\n{input}\\n### 回答\\n\"\"\"\n",
+    "\n",
+    "    inputs = tokenizer([prompt], return_tensors = \"pt\").to(model.device)\n",
+    "\n",
+    "    outputs = model.generate(\n",
+    "        **inputs,\n",
+    "        max_new_tokens=1024,\n",
+    "        use_cache = True, \n",
+    "        do_sample=False, \n",
+    "        repetition_penalty=1.2\n",
+    "      )\n",
+    "    prediction = tokenizer.decode(outputs[0], skip_special_tokens=True).split('\\n### 回答')[-1]\n",
+    "    \n",
+    "    results.append({\"task_id\": dt['task_id'], \"input\": input, \"output\": prediction})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "9a18a4f8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "json_file_id = re.sub(\".*/\", \"\", adapter_id)\n",
+    "with open(f\"{json_file_id}_output.jsonl\", 'w', encoding='utf-8') as f:\n",
+    "    for result in results:\n",
+    "        json.dump(result, f, ensure_ascii=False)\n",
+    "        f.write('\\n')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a2ebf493",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "unsloth",
+   "language": "python",
+   "name": "unsloth"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}