{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "138889b92720ce2e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-04-30T15:08:02.398435Z",
     "start_time": "2024-04-30T15:08:02.194901Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>runname</th>\n",
       "      <th>seed</th>\n",
       "      <th>steps</th>\n",
       "      <th>agg_score</th>\n",
       "      <th>commonsense_qa/acc</th>\n",
       "      <th>commonsense_qa/acc_norm</th>\n",
       "      <th>hellaswag/acc</th>\n",
       "      <th>hellaswag/acc_norm</th>\n",
       "      <th>openbookqa/acc</th>\n",
       "      <th>openbookqa/acc_norm</th>\n",
       "      <th>...</th>\n",
       "      <th>siqa/acc</th>\n",
       "      <th>siqa/acc_norm</th>\n",
       "      <th>winogrande/acc</th>\n",
       "      <th>winogrande/acc_norm</th>\n",
       "      <th>sciq/acc</th>\n",
       "      <th>sciq/acc_norm</th>\n",
       "      <th>arc/acc</th>\n",
       "      <th>arc/acc_norm</th>\n",
       "      <th>mmlu/acc</th>\n",
       "      <th>mmlu/acc_norm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>big-run-refinedweb</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0.330893</td>\n",
       "      <td>0.186</td>\n",
       "      <td>0.233</td>\n",
       "      <td>0.272</td>\n",
       "      <td>0.258</td>\n",
       "      <td>0.166</td>\n",
       "      <td>0.286</td>\n",
       "      <td>...</td>\n",
       "      <td>0.367</td>\n",
       "      <td>0.362</td>\n",
       "      <td>0.516</td>\n",
       "      <td>0.497</td>\n",
       "      <td>0.208</td>\n",
       "      <td>0.202</td>\n",
       "      <td>0.2195</td>\n",
       "      <td>0.2510</td>\n",
       "      <td>0.230294</td>\n",
       "      <td>0.250147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>big-run-refinedweb</td>\n",
       "      <td>6</td>\n",
       "      <td>1000</td>\n",
       "      <td>0.353481</td>\n",
       "      <td>0.233</td>\n",
       "      <td>0.253</td>\n",
       "      <td>0.288</td>\n",
       "      <td>0.276</td>\n",
       "      <td>0.120</td>\n",
       "      <td>0.256</td>\n",
       "      <td>...</td>\n",
       "      <td>0.365</td>\n",
       "      <td>0.398</td>\n",
       "      <td>0.502</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.582</td>\n",
       "      <td>0.528</td>\n",
       "      <td>0.2650</td>\n",
       "      <td>0.2900</td>\n",
       "      <td>0.240583</td>\n",
       "      <td>0.252852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>big-run-refinedweb</td>\n",
       "      <td>6</td>\n",
       "      <td>2000</td>\n",
       "      <td>0.376461</td>\n",
       "      <td>0.282</td>\n",
       "      <td>0.280</td>\n",
       "      <td>0.315</td>\n",
       "      <td>0.328</td>\n",
       "      <td>0.154</td>\n",
       "      <td>0.284</td>\n",
       "      <td>...</td>\n",
       "      <td>0.368</td>\n",
       "      <td>0.390</td>\n",
       "      <td>0.511</td>\n",
       "      <td>0.498</td>\n",
       "      <td>0.683</td>\n",
       "      <td>0.590</td>\n",
       "      <td>0.3055</td>\n",
       "      <td>0.3170</td>\n",
       "      <td>0.245067</td>\n",
       "      <td>0.261686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>big-run-refinedweb</td>\n",
       "      <td>6</td>\n",
       "      <td>3000</td>\n",
       "      <td>0.387825</td>\n",
       "      <td>0.282</td>\n",
       "      <td>0.287</td>\n",
       "      <td>0.331</td>\n",
       "      <td>0.350</td>\n",
       "      <td>0.152</td>\n",
       "      <td>0.306</td>\n",
       "      <td>...</td>\n",
       "      <td>0.376</td>\n",
       "      <td>0.386</td>\n",
       "      <td>0.512</td>\n",
       "      <td>0.495</td>\n",
       "      <td>0.748</td>\n",
       "      <td>0.646</td>\n",
       "      <td>0.3210</td>\n",
       "      <td>0.3410</td>\n",
       "      <td>0.250268</td>\n",
       "      <td>0.266600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>big-run-refinedweb</td>\n",
       "      <td>6</td>\n",
       "      <td>4000</td>\n",
       "      <td>0.398105</td>\n",
       "      <td>0.310</td>\n",
       "      <td>0.318</td>\n",
       "      <td>0.340</td>\n",
       "      <td>0.389</td>\n",
       "      <td>0.168</td>\n",
       "      <td>0.306</td>\n",
       "      <td>...</td>\n",
       "      <td>0.371</td>\n",
       "      <td>0.392</td>\n",
       "      <td>0.513</td>\n",
       "      <td>0.495</td>\n",
       "      <td>0.736</td>\n",
       "      <td>0.634</td>\n",
       "      <td>0.3305</td>\n",
       "      <td>0.3425</td>\n",
       "      <td>0.250732</td>\n",
       "      <td>0.268341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1339</th>\n",
       "      <td>big-run-url_dedups_lowercase_char_length</td>\n",
       "      <td>6</td>\n",
       "      <td>163000</td>\n",
       "      <td>0.477694</td>\n",
       "      <td>0.396</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.477</td>\n",
       "      <td>0.578</td>\n",
       "      <td>0.226</td>\n",
       "      <td>0.354</td>\n",
       "      <td>...</td>\n",
       "      <td>0.408</td>\n",
       "      <td>0.415</td>\n",
       "      <td>0.562</td>\n",
       "      <td>0.548</td>\n",
       "      <td>0.879</td>\n",
       "      <td>0.817</td>\n",
       "      <td>0.4655</td>\n",
       "      <td>0.4540</td>\n",
       "      <td>0.303672</td>\n",
       "      <td>0.325554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1340</th>\n",
       "      <td>big-run-url_dedups_lowercase_char_length</td>\n",
       "      <td>6</td>\n",
       "      <td>164000</td>\n",
       "      <td>0.476591</td>\n",
       "      <td>0.396</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.478</td>\n",
       "      <td>0.581</td>\n",
       "      <td>0.228</td>\n",
       "      <td>0.342</td>\n",
       "      <td>...</td>\n",
       "      <td>0.417</td>\n",
       "      <td>0.414</td>\n",
       "      <td>0.555</td>\n",
       "      <td>0.544</td>\n",
       "      <td>0.883</td>\n",
       "      <td>0.827</td>\n",
       "      <td>0.4600</td>\n",
       "      <td>0.4570</td>\n",
       "      <td>0.306406</td>\n",
       "      <td>0.329724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1341</th>\n",
       "      <td>big-run-url_dedups_lowercase_char_length</td>\n",
       "      <td>6</td>\n",
       "      <td>165000</td>\n",
       "      <td>0.478964</td>\n",
       "      <td>0.405</td>\n",
       "      <td>0.388</td>\n",
       "      <td>0.474</td>\n",
       "      <td>0.583</td>\n",
       "      <td>0.230</td>\n",
       "      <td>0.362</td>\n",
       "      <td>...</td>\n",
       "      <td>0.414</td>\n",
       "      <td>0.412</td>\n",
       "      <td>0.562</td>\n",
       "      <td>0.541</td>\n",
       "      <td>0.881</td>\n",
       "      <td>0.826</td>\n",
       "      <td>0.4545</td>\n",
       "      <td>0.4465</td>\n",
       "      <td>0.304121</td>\n",
       "      <td>0.327213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1342</th>\n",
       "      <td>big-run-url_dedups_lowercase_char_length</td>\n",
       "      <td>6</td>\n",
       "      <td>166000</td>\n",
       "      <td>0.477467</td>\n",
       "      <td>0.398</td>\n",
       "      <td>0.381</td>\n",
       "      <td>0.470</td>\n",
       "      <td>0.579</td>\n",
       "      <td>0.234</td>\n",
       "      <td>0.354</td>\n",
       "      <td>...</td>\n",
       "      <td>0.413</td>\n",
       "      <td>0.411</td>\n",
       "      <td>0.554</td>\n",
       "      <td>0.544</td>\n",
       "      <td>0.887</td>\n",
       "      <td>0.831</td>\n",
       "      <td>0.4625</td>\n",
       "      <td>0.4565</td>\n",
       "      <td>0.305855</td>\n",
       "      <td>0.328240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1343</th>\n",
       "      <td>big-run-url_dedups_lowercase_char_length</td>\n",
       "      <td>6</td>\n",
       "      <td>167000</td>\n",
       "      <td>0.476630</td>\n",
       "      <td>0.398</td>\n",
       "      <td>0.370</td>\n",
       "      <td>0.477</td>\n",
       "      <td>0.577</td>\n",
       "      <td>0.244</td>\n",
       "      <td>0.354</td>\n",
       "      <td>...</td>\n",
       "      <td>0.413</td>\n",
       "      <td>0.414</td>\n",
       "      <td>0.553</td>\n",
       "      <td>0.540</td>\n",
       "      <td>0.879</td>\n",
       "      <td>0.825</td>\n",
       "      <td>0.4660</td>\n",
       "      <td>0.4565</td>\n",
       "      <td>0.307940</td>\n",
       "      <td>0.328538</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1344 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       runname  seed   steps  agg_score  \\\n",
       "0                           big-run-refinedweb     6       0   0.330893   \n",
       "1                           big-run-refinedweb     6    1000   0.353481   \n",
       "2                           big-run-refinedweb     6    2000   0.376461   \n",
       "3                           big-run-refinedweb     6    3000   0.387825   \n",
       "4                           big-run-refinedweb     6    4000   0.398105   \n",
       "...                                        ...   ...     ...        ...   \n",
       "1339  big-run-url_dedups_lowercase_char_length     6  163000   0.477694   \n",
       "1340  big-run-url_dedups_lowercase_char_length     6  164000   0.476591   \n",
       "1341  big-run-url_dedups_lowercase_char_length     6  165000   0.478964   \n",
       "1342  big-run-url_dedups_lowercase_char_length     6  166000   0.477467   \n",
       "1343  big-run-url_dedups_lowercase_char_length     6  167000   0.476630   \n",
       "\n",
       "      commonsense_qa/acc  commonsense_qa/acc_norm  hellaswag/acc  \\\n",
       "0                  0.186                    0.233          0.272   \n",
       "1                  0.233                    0.253          0.288   \n",
       "2                  0.282                    0.280          0.315   \n",
       "3                  0.282                    0.287          0.331   \n",
       "4                  0.310                    0.318          0.340   \n",
       "...                  ...                      ...            ...   \n",
       "1339               0.396                    0.375          0.477   \n",
       "1340               0.396                    0.375          0.478   \n",
       "1341               0.405                    0.388          0.474   \n",
       "1342               0.398                    0.381          0.470   \n",
       "1343               0.398                    0.370          0.477   \n",
       "\n",
       "      hellaswag/acc_norm  openbookqa/acc  openbookqa/acc_norm  ...  siqa/acc  \\\n",
       "0                  0.258           0.166                0.286  ...     0.367   \n",
       "1                  0.276           0.120                0.256  ...     0.365   \n",
       "2                  0.328           0.154                0.284  ...     0.368   \n",
       "3                  0.350           0.152                0.306  ...     0.376   \n",
       "4                  0.389           0.168                0.306  ...     0.371   \n",
       "...                  ...             ...                  ...  ...       ...   \n",
       "1339               0.578           0.226                0.354  ...     0.408   \n",
       "1340               0.581           0.228                0.342  ...     0.417   \n",
       "1341               0.583           0.230                0.362  ...     0.414   \n",
       "1342               0.579           0.234                0.354  ...     0.413   \n",
       "1343               0.577           0.244                0.354  ...     0.413   \n",
       "\n",
       "      siqa/acc_norm  winogrande/acc  winogrande/acc_norm  sciq/acc  \\\n",
       "0             0.362           0.516                0.497     0.208   \n",
       "1             0.398           0.502                0.500     0.582   \n",
       "2             0.390           0.511                0.498     0.683   \n",
       "3             0.386           0.512                0.495     0.748   \n",
       "4             0.392           0.513                0.495     0.736   \n",
       "...             ...             ...                  ...       ...   \n",
       "1339          0.415           0.562                0.548     0.879   \n",
       "1340          0.414           0.555                0.544     0.883   \n",
       "1341          0.412           0.562                0.541     0.881   \n",
       "1342          0.411           0.554                0.544     0.887   \n",
       "1343          0.414           0.553                0.540     0.879   \n",
       "\n",
       "      sciq/acc_norm  arc/acc  arc/acc_norm  mmlu/acc  mmlu/acc_norm  \n",
       "0             0.202   0.2195        0.2510  0.230294       0.250147  \n",
       "1             0.528   0.2650        0.2900  0.240583       0.252852  \n",
       "2             0.590   0.3055        0.3170  0.245067       0.261686  \n",
       "3             0.646   0.3210        0.3410  0.250268       0.266600  \n",
       "4             0.634   0.3305        0.3425  0.250732       0.268341  \n",
       "...             ...      ...           ...       ...            ...  \n",
       "1339          0.817   0.4655        0.4540  0.303672       0.325554  \n",
       "1340          0.827   0.4600        0.4570  0.306406       0.329724  \n",
       "1341          0.826   0.4545        0.4465  0.304121       0.327213  \n",
       "1342          0.831   0.4625        0.4565  0.305855       0.328240  \n",
       "1343          0.825   0.4660        0.4565  0.307940       0.328538  \n",
       "\n",
       "[1344 rows x 22 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from matplotlib.figure import Figure\n",
    "\n",
    "df = pd.read_csv(\"../src_data/diff_dedup_attempts.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "874ab88a573cd443",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-13T13:56:19.453420Z",
     "start_time": "2024-05-13T13:56:19.450850Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['big-run-refinedweb',\n",
       " 'big-run-sampled_cross_minhash_dump',\n",
       " 'big-run-sampled_full_filtered_no_dedup',\n",
       " 'big-run-sampled_full_imh_linededup',\n",
       " 'big-run-sampled_full_ind_minhash',\n",
       " 'big-run-sampled_line_dedup_3lines2',\n",
       " 'big-run-sampled_line_dedup_min_words',\n",
       " 'big-run-url_dedups_lowercase_char_length']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.unique(df[\"runname\"]).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b610f43caefdf01",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-13T14:00:46.578560Z",
     "start_time": "2024-05-13T14:00:46.576167Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "runs_mapping = {\n",
    "    \"big-run-refinedweb\": \"RefinedWeb\",\n",
    "    \"big-run-sampled_cross_minhash_dump\": \"FineWeb full MinHash\",\n",
    "    \"big-run-sampled_full_filtered_no_dedup\": \"FineWeb filtered only\",\n",
    "    \"big-run-sampled_full_ind_minhash\": \"FineWeb independent MinHash\",\n",
    "    \"big-run-sampled_full_imh_linededup\": \"FineWeb line dedup\",\n",
    "    \"big-run-sampled_line_dedup_3lines2\": \"FineWeb 3-line dedup\",\n",
    "    \"big-run-sampled_line_dedup_min_words\": \"FineWeb line dedup w/ min words\",\n",
    "    \"big-run-url_dedups_lowercase_char_length\": \"FineWeb URL dedup\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-13T14:04:41.777032Z",
     "start_time": "2024-05-13T14:04:41.536919Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "from matplotlib import pyplot as plt\n",
    "metrics = ['agg_score', 'commonsense_qa/acc_norm', 'hellaswag/acc_norm', 'openbookqa/acc_norm', 'piqa/acc_norm',\n",
    "                   'siqa/acc_norm', 'winogrande/acc_norm', 'arc/acc_norm', 'mmlu/acc_norm']\n",
    "\n",
    "def normalize_runname(runname):\n",
    "    return runname.replace(\"/\", \"_\")\n",
    "\n",
    "grouped = (\n",
    "    df.groupby([\"runname\", \"steps\"])\n",
    "    .agg(\n",
    "        {\n",
    "            key: \"mean\" for key in metrics\n",
    "        }\n",
    "    )\n",
    "    .reset_index()\n",
    ")\n",
    "\n",
    "file_id=\"../assets/data/plots/dedup_attempts\"\n",
    "files = {}\n",
    "for metric in metrics:\n",
    "    datas = {}\n",
    "    for name, group in grouped.groupby(\"runname\"):\n",
    "        group = group[[\"steps\", metric]].sort_values(by=\"steps\")\n",
    "        group = group.set_index(\"steps\")\n",
    "        rolling_avg = group\n",
    "        # rolling_avg = group.rolling(wjjjjjjjjjjjjjindow=5).mean()\n",
    "        datas[name] = {\n",
    "            \"x\": (rolling_avg.index * 2048 * 1024 * 1e-9).tolist(),\n",
    "            \"y\": rolling_avg[metric].tolist(),\n",
    "            \"label\": runs_mapping[name],\n",
    "        }\n",
    "    # Sort the datata based on the steps\n",
    "    datas = {k: v for k, v in sorted(datas.items(), key=lambda x: -x[1][\"y\"][-1])}\n",
    "    # Create a folder\n",
    "    os.makedirs(f\"{file_id}\", exist_ok=True)\n",
    "    with open(f\"{file_id}/{normalize_runname(metric)}.json\", \"w\") as f:\n",
    "        json.dump({\n",
    "            \"data\": datas,\n",
    "            \"layout\": {\n",
    "                \"title\": {\n",
    "                    \"text\": \"Attempting to further globally dedup worsened perf\"\n",
    "                },\n",
    "            }\n",
    "        }, f)\n",
    "    files[metric] = {\"file\": f\"{normalize_runname(metric)}.json\"}\n",
    "# Create index\n",
    "with open(f\"{file_id}/index.json\", \"w\") as f:\n",
    "    json.dump({\n",
    "        \"files\": files,\n",
    "        \"settings\": {\n",
    "            \"defaultMetric\": \"agg_score\",\n",
    "            \"slider\":{\"min\":0,\"max\":30,\"default\":5}\n",
    "        }\n",
    "    }, f)\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "af28ebbd054cdc33",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-04-30T15:08:02.522543Z",
     "start_time": "2024-04-30T15:08:02.520569Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}