{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns \n",
    "from matplotlib import pyplot as plt\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sft_log_file = '../logs/sft_train_log_20231211-2250.csv'\n",
    "dpo_log_file = '../logs/dpo_train_log_20231213-0214.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>epoch</th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>loss</th>\n",
       "      <th>step</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.400000e-08</td>\n",
       "      <td>2.5986</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.400000e-06</td>\n",
       "      <td>2.6353</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.01</td>\n",
       "      <td>2.800000e-06</td>\n",
       "      <td>2.4905</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.01</td>\n",
       "      <td>4.200000e-06</td>\n",
       "      <td>2.3610</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.01</td>\n",
       "      <td>5.600000e-06</td>\n",
       "      <td>2.2837</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  epoch  learning_rate    loss  step\n",
       "0           0   0.00   1.400000e-08  2.5986     1\n",
       "1           1   0.00   1.400000e-06  2.6353   100\n",
       "2           2   0.01   2.800000e-06  2.4905   200\n",
       "3           3   0.01   4.200000e-06  2.3610   300\n",
       "4           4   0.01   5.600000e-06  2.2837   400"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sft_df = pd.read_csv(sft_log_file)\n",
    "dpo_df = pd.read_csv(dpo_log_file)\n",
    "sft_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plt.title('learning_rate')\n",
    "sns.lineplot(\n",
    "    x=\"step\", \n",
    "    y=\"learning_rate\", \n",
    "    data=sft_df,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.title('sft loss')\n",
    "sns.lineplot(\n",
    "    x=\"step\", \n",
    "    y=\"loss\", \n",
    "    color='dodgerblue',\n",
    "    data=sft_df,\n",
    "    )\n",
    "plt.savefig('../img/sft_loss.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>epoch</th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>logits/chosen</th>\n",
       "      <th>logits/rejected</th>\n",
       "      <th>logps/chosen</th>\n",
       "      <th>logps/rejected</th>\n",
       "      <th>loss</th>\n",
       "      <th>rewards/accuracies</th>\n",
       "      <th>rewards/chosen</th>\n",
       "      <th>rewards/margins</th>\n",
       "      <th>rewards/rejected</th>\n",
       "      <th>step</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000e-08</td>\n",
       "      <td>-3.525447</td>\n",
       "      <td>-3.550683</td>\n",
       "      <td>-256.702698</td>\n",
       "      <td>-143.308243</td>\n",
       "      <td>0.7689</td>\n",
       "      <td>0.437500</td>\n",
       "      <td>-0.044875</td>\n",
       "      <td>-0.072844</td>\n",
       "      <td>0.027969</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.01</td>\n",
       "      <td>2.000000e-07</td>\n",
       "      <td>-3.509013</td>\n",
       "      <td>-3.557282</td>\n",
       "      <td>-270.281708</td>\n",
       "      <td>-150.850433</td>\n",
       "      <td>0.7438</td>\n",
       "      <td>0.486842</td>\n",
       "      <td>0.002034</td>\n",
       "      <td>-0.020194</td>\n",
       "      <td>0.022228</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.01</td>\n",
       "      <td>4.000000e-07</td>\n",
       "      <td>-3.509622</td>\n",
       "      <td>-3.544898</td>\n",
       "      <td>-286.783966</td>\n",
       "      <td>-162.946915</td>\n",
       "      <td>0.7038</td>\n",
       "      <td>0.529688</td>\n",
       "      <td>0.024229</td>\n",
       "      <td>0.046643</td>\n",
       "      <td>-0.022414</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.02</td>\n",
       "      <td>6.000000e-07</td>\n",
       "      <td>-3.521220</td>\n",
       "      <td>-3.554179</td>\n",
       "      <td>-267.424896</td>\n",
       "      <td>-151.984573</td>\n",
       "      <td>0.7218</td>\n",
       "      <td>0.507812</td>\n",
       "      <td>0.004973</td>\n",
       "      <td>0.008775</td>\n",
       "      <td>-0.003803</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.03</td>\n",
       "      <td>8.000000e-07</td>\n",
       "      <td>-3.513215</td>\n",
       "      <td>-3.551011</td>\n",
       "      <td>-281.538208</td>\n",
       "      <td>-157.784546</td>\n",
       "      <td>0.6995</td>\n",
       "      <td>0.548437</td>\n",
       "      <td>0.057179</td>\n",
       "      <td>0.069537</td>\n",
       "      <td>-0.012358</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  epoch  learning_rate  logits/chosen  logits/rejected  \\\n",
       "0           0   0.00   1.000000e-08      -3.525447        -3.550683   \n",
       "1           1   0.01   2.000000e-07      -3.509013        -3.557282   \n",
       "2           2   0.01   4.000000e-07      -3.509622        -3.544898   \n",
       "3           3   0.02   6.000000e-07      -3.521220        -3.554179   \n",
       "4           4   0.03   8.000000e-07      -3.513215        -3.551011   \n",
       "\n",
       "   logps/chosen  logps/rejected    loss  rewards/accuracies  rewards/chosen  \\\n",
       "0   -256.702698     -143.308243  0.7689            0.437500       -0.044875   \n",
       "1   -270.281708     -150.850433  0.7438            0.486842        0.002034   \n",
       "2   -286.783966     -162.946915  0.7038            0.529688        0.024229   \n",
       "3   -267.424896     -151.984573  0.7218            0.507812        0.004973   \n",
       "4   -281.538208     -157.784546  0.6995            0.548437        0.057179   \n",
       "\n",
       "   rewards/margins  rewards/rejected  step  \n",
       "0        -0.072844          0.027969     1  \n",
       "1        -0.020194          0.022228    20  \n",
       "2         0.046643         -0.022414    40  \n",
       "3         0.008775         -0.003803    60  \n",
       "4         0.069537         -0.012358    80  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dpo_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.title('dpo loss')\n",
    "sns.lineplot(\n",
    "    x=\"step\", \n",
    "    y=\"loss\", \n",
    "    color='orange',\n",
    "    data=dpo_df[0: 6000 // 20],  # 只使用了到6000步的checkpoit，后面的有过拟合迹象\n",
    "    )\n",
    "plt.savefig('../img/dpo_loss.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import sys, os\n",
    "root = os.path.realpath('.').replace('\\\\','/').split('/')[0: -1]\n",
    "root = '/'.join(root)\n",
    "sys.path.append(root)\n",
    "\n",
    "from model.infer import ChatBot\n",
    "from config import InferConfig\n",
    "\n",
    "bot = ChatBot(InferConfig())\n",
    "model = bot.model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model parameters size: 210.19 M = 0.21B\n",
      "GPU memory used: 0.40GB\n"
     ]
    }
   ],
   "source": [
    "param_size = sum([p.numel() for p in model.parameters()]) / 1000 / 1000\n",
    "print('model parameters size: {:.2f} M = {:.2f}B'.format( param_size , param_size / 1000))\n",
    "\n",
    "print('GPU memory used: {:.2f}GB'.format(torch.cuda.memory_allocated() / (1024 ** 3)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py310",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}