{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "138889b92720ce2e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-13T14:36:31.336129Z",
     "start_time": "2024-05-13T14:36:31.323847Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>runname</th>\n",
       "      <th>seed</th>\n",
       "      <th>steps</th>\n",
       "      <th>agg_score</th>\n",
       "      <th>commonsense_qa/acc</th>\n",
       "      <th>commonsense_qa/acc_norm</th>\n",
       "      <th>hellaswag/acc</th>\n",
       "      <th>hellaswag/acc_norm</th>\n",
       "      <th>openbookqa/acc</th>\n",
       "      <th>openbookqa/acc_norm</th>\n",
       "      <th>...</th>\n",
       "      <th>siqa/acc</th>\n",
       "      <th>siqa/acc_norm</th>\n",
       "      <th>winogrande/acc</th>\n",
       "      <th>winogrande/acc_norm</th>\n",
       "      <th>sciq/acc</th>\n",
       "      <th>sciq/acc_norm</th>\n",
       "      <th>arc/acc</th>\n",
       "      <th>arc/acc_norm</th>\n",
       "      <th>mmlu/acc</th>\n",
       "      <th>mmlu/acc_norm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>filtering-baseline-2019-18-40gt</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0.330953</td>\n",
       "      <td>0.186</td>\n",
       "      <td>0.233</td>\n",
       "      <td>0.272</td>\n",
       "      <td>0.258</td>\n",
       "      <td>0.166</td>\n",
       "      <td>0.286</td>\n",
       "      <td>...</td>\n",
       "      <td>0.367</td>\n",
       "      <td>0.362</td>\n",
       "      <td>0.516</td>\n",
       "      <td>0.497</td>\n",
       "      <td>0.210</td>\n",
       "      <td>0.202</td>\n",
       "      <td>0.2190</td>\n",
       "      <td>0.2515</td>\n",
       "      <td>0.230285</td>\n",
       "      <td>0.250127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>filtering-baseline-2019-18-40gt</td>\n",
       "      <td>5</td>\n",
       "      <td>1000</td>\n",
       "      <td>0.357474</td>\n",
       "      <td>0.239</td>\n",
       "      <td>0.271</td>\n",
       "      <td>0.297</td>\n",
       "      <td>0.287</td>\n",
       "      <td>0.146</td>\n",
       "      <td>0.260</td>\n",
       "      <td>...</td>\n",
       "      <td>0.365</td>\n",
       "      <td>0.396</td>\n",
       "      <td>0.503</td>\n",
       "      <td>0.486</td>\n",
       "      <td>0.568</td>\n",
       "      <td>0.502</td>\n",
       "      <td>0.2665</td>\n",
       "      <td>0.2855</td>\n",
       "      <td>0.242526</td>\n",
       "      <td>0.253291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>filtering-baseline-2019-18-40gt</td>\n",
       "      <td>5</td>\n",
       "      <td>2000</td>\n",
       "      <td>0.377436</td>\n",
       "      <td>0.280</td>\n",
       "      <td>0.284</td>\n",
       "      <td>0.321</td>\n",
       "      <td>0.332</td>\n",
       "      <td>0.134</td>\n",
       "      <td>0.268</td>\n",
       "      <td>...</td>\n",
       "      <td>0.368</td>\n",
       "      <td>0.399</td>\n",
       "      <td>0.519</td>\n",
       "      <td>0.502</td>\n",
       "      <td>0.686</td>\n",
       "      <td>0.590</td>\n",
       "      <td>0.3030</td>\n",
       "      <td>0.3215</td>\n",
       "      <td>0.245745</td>\n",
       "      <td>0.260988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>filtering-baseline-2019-18-40gt</td>\n",
       "      <td>5</td>\n",
       "      <td>3000</td>\n",
       "      <td>0.387994</td>\n",
       "      <td>0.277</td>\n",
       "      <td>0.291</td>\n",
       "      <td>0.339</td>\n",
       "      <td>0.359</td>\n",
       "      <td>0.132</td>\n",
       "      <td>0.280</td>\n",
       "      <td>...</td>\n",
       "      <td>0.394</td>\n",
       "      <td>0.404</td>\n",
       "      <td>0.520</td>\n",
       "      <td>0.503</td>\n",
       "      <td>0.721</td>\n",
       "      <td>0.622</td>\n",
       "      <td>0.3210</td>\n",
       "      <td>0.3385</td>\n",
       "      <td>0.250427</td>\n",
       "      <td>0.264451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>filtering-baseline-2019-18-40gt</td>\n",
       "      <td>5</td>\n",
       "      <td>4000</td>\n",
       "      <td>0.396110</td>\n",
       "      <td>0.299</td>\n",
       "      <td>0.315</td>\n",
       "      <td>0.340</td>\n",
       "      <td>0.366</td>\n",
       "      <td>0.158</td>\n",
       "      <td>0.286</td>\n",
       "      <td>...</td>\n",
       "      <td>0.376</td>\n",
       "      <td>0.399</td>\n",
       "      <td>0.515</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.739</td>\n",
       "      <td>0.620</td>\n",
       "      <td>0.3320</td>\n",
       "      <td>0.3445</td>\n",
       "      <td>0.256134</td>\n",
       "      <td>0.270382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250</th>\n",
       "      <td>sm-baseline-c4</td>\n",
       "      <td>6</td>\n",
       "      <td>10000</td>\n",
       "      <td>0.430443</td>\n",
       "      <td>0.335</td>\n",
       "      <td>0.326</td>\n",
       "      <td>0.379</td>\n",
       "      <td>0.474</td>\n",
       "      <td>0.176</td>\n",
       "      <td>0.340</td>\n",
       "      <td>...</td>\n",
       "      <td>0.385</td>\n",
       "      <td>0.406</td>\n",
       "      <td>0.525</td>\n",
       "      <td>0.523</td>\n",
       "      <td>0.767</td>\n",
       "      <td>0.675</td>\n",
       "      <td>0.3765</td>\n",
       "      <td>0.3750</td>\n",
       "      <td>0.269139</td>\n",
       "      <td>0.280545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>251</th>\n",
       "      <td>sm-baseline-c4</td>\n",
       "      <td>6</td>\n",
       "      <td>11000</td>\n",
       "      <td>0.430776</td>\n",
       "      <td>0.341</td>\n",
       "      <td>0.323</td>\n",
       "      <td>0.391</td>\n",
       "      <td>0.481</td>\n",
       "      <td>0.192</td>\n",
       "      <td>0.346</td>\n",
       "      <td>...</td>\n",
       "      <td>0.390</td>\n",
       "      <td>0.405</td>\n",
       "      <td>0.531</td>\n",
       "      <td>0.515</td>\n",
       "      <td>0.766</td>\n",
       "      <td>0.676</td>\n",
       "      <td>0.3775</td>\n",
       "      <td>0.3770</td>\n",
       "      <td>0.266895</td>\n",
       "      <td>0.281210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>252</th>\n",
       "      <td>sm-baseline-c4</td>\n",
       "      <td>6</td>\n",
       "      <td>12000</td>\n",
       "      <td>0.430352</td>\n",
       "      <td>0.340</td>\n",
       "      <td>0.319</td>\n",
       "      <td>0.392</td>\n",
       "      <td>0.475</td>\n",
       "      <td>0.192</td>\n",
       "      <td>0.342</td>\n",
       "      <td>...</td>\n",
       "      <td>0.377</td>\n",
       "      <td>0.395</td>\n",
       "      <td>0.528</td>\n",
       "      <td>0.518</td>\n",
       "      <td>0.785</td>\n",
       "      <td>0.688</td>\n",
       "      <td>0.3755</td>\n",
       "      <td>0.3840</td>\n",
       "      <td>0.267159</td>\n",
       "      <td>0.279819</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253</th>\n",
       "      <td>sm-baseline-c4</td>\n",
       "      <td>6</td>\n",
       "      <td>13000</td>\n",
       "      <td>0.432136</td>\n",
       "      <td>0.339</td>\n",
       "      <td>0.326</td>\n",
       "      <td>0.395</td>\n",
       "      <td>0.477</td>\n",
       "      <td>0.198</td>\n",
       "      <td>0.348</td>\n",
       "      <td>...</td>\n",
       "      <td>0.390</td>\n",
       "      <td>0.405</td>\n",
       "      <td>0.529</td>\n",
       "      <td>0.518</td>\n",
       "      <td>0.785</td>\n",
       "      <td>0.682</td>\n",
       "      <td>0.3780</td>\n",
       "      <td>0.3825</td>\n",
       "      <td>0.269719</td>\n",
       "      <td>0.281585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254</th>\n",
       "      <td>sm-baseline-c4</td>\n",
       "      <td>6</td>\n",
       "      <td>13500</td>\n",
       "      <td>0.433866</td>\n",
       "      <td>0.344</td>\n",
       "      <td>0.328</td>\n",
       "      <td>0.394</td>\n",
       "      <td>0.484</td>\n",
       "      <td>0.198</td>\n",
       "      <td>0.334</td>\n",
       "      <td>...</td>\n",
       "      <td>0.388</td>\n",
       "      <td>0.406</td>\n",
       "      <td>0.531</td>\n",
       "      <td>0.523</td>\n",
       "      <td>0.778</td>\n",
       "      <td>0.682</td>\n",
       "      <td>0.3795</td>\n",
       "      <td>0.3845</td>\n",
       "      <td>0.269601</td>\n",
       "      <td>0.284425</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>255 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             runname  seed  steps  agg_score  \\\n",
       "0    filtering-baseline-2019-18-40gt     5      0   0.330953   \n",
       "1    filtering-baseline-2019-18-40gt     5   1000   0.357474   \n",
       "2    filtering-baseline-2019-18-40gt     5   2000   0.377436   \n",
       "3    filtering-baseline-2019-18-40gt     5   3000   0.387994   \n",
       "4    filtering-baseline-2019-18-40gt     5   4000   0.396110   \n",
       "..                               ...   ...    ...        ...   \n",
       "250                   sm-baseline-c4     6  10000   0.430443   \n",
       "251                   sm-baseline-c4     6  11000   0.430776   \n",
       "252                   sm-baseline-c4     6  12000   0.430352   \n",
       "253                   sm-baseline-c4     6  13000   0.432136   \n",
       "254                   sm-baseline-c4     6  13500   0.433866   \n",
       "\n",
       "     commonsense_qa/acc  commonsense_qa/acc_norm  hellaswag/acc  \\\n",
       "0                 0.186                    0.233          0.272   \n",
       "1                 0.239                    0.271          0.297   \n",
       "2                 0.280                    0.284          0.321   \n",
       "3                 0.277                    0.291          0.339   \n",
       "4                 0.299                    0.315          0.340   \n",
       "..                  ...                      ...            ...   \n",
       "250               0.335                    0.326          0.379   \n",
       "251               0.341                    0.323          0.391   \n",
       "252               0.340                    0.319          0.392   \n",
       "253               0.339                    0.326          0.395   \n",
       "254               0.344                    0.328          0.394   \n",
       "\n",
       "     hellaswag/acc_norm  openbookqa/acc  openbookqa/acc_norm  ...  siqa/acc  \\\n",
       "0                 0.258           0.166                0.286  ...     0.367   \n",
       "1                 0.287           0.146                0.260  ...     0.365   \n",
       "2                 0.332           0.134                0.268  ...     0.368   \n",
       "3                 0.359           0.132                0.280  ...     0.394   \n",
       "4                 0.366           0.158                0.286  ...     0.376   \n",
       "..                  ...             ...                  ...  ...       ...   \n",
       "250               0.474           0.176                0.340  ...     0.385   \n",
       "251               0.481           0.192                0.346  ...     0.390   \n",
       "252               0.475           0.192                0.342  ...     0.377   \n",
       "253               0.477           0.198                0.348  ...     0.390   \n",
       "254               0.484           0.198                0.334  ...     0.388   \n",
       "\n",
       "     siqa/acc_norm  winogrande/acc  winogrande/acc_norm  sciq/acc  \\\n",
       "0            0.362           0.516                0.497     0.210   \n",
       "1            0.396           0.503                0.486     0.568   \n",
       "2            0.399           0.519                0.502     0.686   \n",
       "3            0.404           0.520                0.503     0.721   \n",
       "4            0.399           0.515                0.500     0.739   \n",
       "..             ...             ...                  ...       ...   \n",
       "250          0.406           0.525                0.523     0.767   \n",
       "251          0.405           0.531                0.515     0.766   \n",
       "252          0.395           0.528                0.518     0.785   \n",
       "253          0.405           0.529                0.518     0.785   \n",
       "254          0.406           0.531                0.523     0.778   \n",
       "\n",
       "     sciq/acc_norm  arc/acc  arc/acc_norm  mmlu/acc  mmlu/acc_norm  \n",
       "0            0.202   0.2190        0.2515  0.230285       0.250127  \n",
       "1            0.502   0.2665        0.2855  0.242526       0.253291  \n",
       "2            0.590   0.3030        0.3215  0.245745       0.260988  \n",
       "3            0.622   0.3210        0.3385  0.250427       0.264451  \n",
       "4            0.620   0.3320        0.3445  0.256134       0.270382  \n",
       "..             ...      ...           ...       ...            ...  \n",
       "250          0.675   0.3765        0.3750  0.269139       0.280545  \n",
       "251          0.676   0.3775        0.3770  0.266895       0.281210  \n",
       "252          0.688   0.3755        0.3840  0.267159       0.279819  \n",
       "253          0.682   0.3780        0.3825  0.269719       0.281585  \n",
       "254          0.682   0.3795        0.3845  0.269601       0.284425  \n",
       "\n",
       "[255 rows x 22 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from matplotlib.figure import Figure\n",
    "\n",
    "df = pd.read_csv(\"../src_data/c4-filters.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "839a06a71d9183e5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-13T14:36:32.338012Z",
     "start_time": "2024-05-13T14:36:32.335209Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['filtering-baseline-2019-18-40gt',\n",
       " 'filtering-baseline-2019-18-60gt',\n",
       " 'filtering-c4-all-except-terminal_punct',\n",
       " 'filtering-c4-all',\n",
       " 'filtering-c4-curly_bracket',\n",
       " 'filtering-c4-terminal_punct',\n",
       " 'filtering-c4-word_lengths',\n",
       " 'sm-baseline-c4']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.unique(df[\"runname\"]).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b610f43caefdf01",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-13T16:06:36.968532Z",
     "start_time": "2024-05-13T16:06:36.966172Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "runs_mapping = {\n",
    " # 'filtering-baseline-2019-18-40gt': \"baseline\",\n",
    " 'filtering-baseline-2019-18-60gt': \"baseline\",\n",
    " 'filtering-c4-curly_bracket': \"curly_bracket filter\",\n",
    " 'filtering-c4-terminal_punct': \"terminal_punct filter\",\n",
    " 'filtering-c4-word_lengths': \"word_lengths filter\",\n",
    " 'filtering-c4-all': \"All filters\",\n",
    " 'filtering-c4-all-except-terminal_punct': \"All filters except terminal_punct\",\n",
    " 'sm-baseline-c4': \"C4\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-13T16:06:37.459935Z",
     "start_time": "2024-05-13T16:06:37.181024Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "\n",
    "import json\n",
    "import os\n",
    "from matplotlib import pyplot as plt\n",
    "metrics = ['agg_score', 'commonsense_qa/acc_norm', 'hellaswag/acc_norm', 'openbookqa/acc_norm', 'piqa/acc_norm',\n",
    "                   'siqa/acc_norm', 'winogrande/acc_norm', 'arc/acc_norm', 'mmlu/acc_norm']\n",
    "\n",
    "def normalize_runname(runname):\n",
    "    return runname.replace(\"/\", \"_\")\n",
    "\n",
    "grouped = (\n",
    "    df.groupby([\"runname\", \"steps\"])\n",
    "    .agg(\n",
    "        {\n",
    "            key: \"mean\" for key in metrics\n",
    "        }\n",
    "    )\n",
    "    .reset_index()\n",
    ")\n",
    "\n",
    "file_id=\"../assets/data/plots/c4_filters_hellaswag\"\n",
    "files = {}\n",
    "for metric in metrics:\n",
    "    datas = {}\n",
    "    for name, group in grouped.groupby(\"runname\"):\n",
    "        if name not in runs_mapping:\n",
    "            continue\n",
    "        group = group[[\"steps\", metric]].sort_values(by=\"steps\")\n",
    "        group = group.set_index(\"steps\")\n",
    "        rolling_avg = group\n",
    "        datas[name] = {\n",
    "            \"x\": (rolling_avg.index * 2048 * 1024 * 1e-9).tolist(),\n",
    "            \"y\": rolling_avg[metric].tolist(),\n",
    "            \"label\": runs_mapping[name],\n",
    "        }\n",
    "    # Sort the datata based on the steps\n",
    "    datas = {k: v for k, v in sorted(datas.items(), key=lambda x: -x[1][\"y\"][-1])}\n",
    "    # Create a folder\n",
    "    os.makedirs(f\"{file_id}\", exist_ok=True)\n",
    "    with open(f\"{file_id}/{normalize_runname(metric)}.json\", \"w\") as f:\n",
    "        json.dump({\n",
    "            \"data\": datas,\n",
    "            \"layout\": {\n",
    "                \"title\": {\n",
    "                    \"text\": \"C4 filtering effect on HellaSwag\"\n",
    "                },\n",
    "            }\n",
    "        }, f)\n",
    "    files[metric] = {\"file\": f\"{normalize_runname(metric)}.json\"}\n",
    "# Create index\n",
    "with open(f\"{file_id}/index.json\", \"w\") as f:\n",
    "    json.dump({\n",
    "        \"files\": files,\n",
    "        \"settings\": {\n",
    "            \"defaultMetric\": \"hellaswag/acc_norm\",\n",
    "            \"slider\":{\"min\":0,\"max\":10,\"default\":3}\n",
    "        }\n",
    "    }, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "af28ebbd054cdc33",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-04-30T12:52:05.836260Z",
     "start_time": "2024-04-30T12:52:05.834381Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}