{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gradio as gr\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running on local URL:  http://127.0.0.1:7868\n",
      "\n",
      "To create a public link, set `share=True` in `launch()`.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><iframe src=\"http://127.0.0.1:7868/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"c:\\Users\\djin\\Miniconda3\\envs\\PH\\lib\\site-packages\\gradio\\routes.py\", line 374, in run_predict\n",
      "    output = await app.get_blocks().process_api(\n",
      "  File \"c:\\Users\\djin\\Miniconda3\\envs\\PH\\lib\\site-packages\\gradio\\blocks.py\", line 1017, in process_api\n",
      "    result = await self.call_function(\n",
      "  File \"c:\\Users\\djin\\Miniconda3\\envs\\PH\\lib\\site-packages\\gradio\\blocks.py\", line 835, in call_function\n",
      "    prediction = await anyio.to_thread.run_sync(\n",
      "  File \"c:\\Users\\djin\\Miniconda3\\envs\\PH\\lib\\site-packages\\anyio\\to_thread.py\", line 31, in run_sync\n",
      "    return await get_asynclib().run_sync_in_worker_thread(\n",
      "  File \"c:\\Users\\djin\\Miniconda3\\envs\\PH\\lib\\site-packages\\anyio\\_backends\\_asyncio.py\", line 937, in run_sync_in_worker_thread\n",
      "    return await future\n",
      "  File \"c:\\Users\\djin\\Miniconda3\\envs\\PH\\lib\\site-packages\\anyio\\_backends\\_asyncio.py\", line 867, in run\n",
      "    result = context.run(func, *args)\n",
      "  File \"C:\\Users\\djin\\AppData\\Local\\Temp/ipykernel_26080/1520398150.py\", line 66, in bootstrap_ci\n",
      "    vals.append(get_severity(bootstrap_sample(conf_mat, k), weights_df))\n",
      "  File \"C:\\Users\\djin\\AppData\\Local\\Temp/ipykernel_26080/1520398150.py\", line 59, in bootstrap_sample\n",
      "    return counts_to_df(random.choices(population=range(num_labels**2), weights=flatten(conf_mat), k=k), num_labels)\n",
      "  File \"c:\\Users\\djin\\Miniconda3\\envs\\PH\\lib\\random.py\", line 502, in choices\n",
      "    raise ValueError('Total of weights must be greater than zero')\n",
      "ValueError: Total of weights must be greater than zero\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import gradio as gr\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "num_labels = 3\n",
    "headers = [\"Benign\",\"C1\",\"C2\"]\n",
    "default_weights = pd.DataFrame([[0,1,2],[1,0,1],[2,1,0]],columns=headers)\n",
    "example_conf_mats = [\n",
    "    pd.DataFrame([[80,10,0],\n",
    "                  [20,80,20],\n",
    "                  [0,10,80]],columns=headers),\n",
    "    pd.DataFrame([[80,10,10],\n",
    "                  [10,80,10],\n",
    "                  [10,10,80]],columns=headers),\n",
    "    pd.DataFrame([[80,10,20],\n",
    "                  [0,80,0],\n",
    "                  [20,10,80]],columns=headers),\n",
    "    pd.DataFrame([[800,100,100],\n",
    "                  [100,800,100],\n",
    "                  [100,100,800]],columns=headers),\n",
    "    pd.DataFrame([[800,100,200],\n",
    "                  [0,800,0],\n",
    "                  [200,100,800]],columns=headers),\n",
    "]\n",
    "\n",
    "def submit_vals(*argv):\n",
    "    argv = list(argv)\n",
    "    weights = np.zeros((num_labels, num_labels))\n",
    "    for i in range(num_labels):\n",
    "        for j in range(num_labels):\n",
    "            if i != j:\n",
    "                weights[i][j] = argv.pop(0)\n",
    "    weights_df = pd.DataFrame(weights, columns=headers)\n",
    "    return weights_df\n",
    "\n",
    "def get_severity(input_df, weights_df):\n",
    "    weights_df.columns = input_df.columns\n",
    "    total = sum(sum(np.array(input_df)))\n",
    "    diag_total = sum(np.diag(input_df))\n",
    "    non_diag_total = total - diag_total\n",
    "    if non_diag_total == 0:\n",
    "        severity = 0\n",
    "    else:\n",
    "        severity = sum(sum(np.array(input_df.multiply(weights_df)))) / non_diag_total\n",
    "    return severity\n",
    "\n",
    "def get_acc(input_df, weights_df):\n",
    "    input_df = input_df.astype(int)\n",
    "    accuracy = 100 * sum(np.diag(input_df)) / sum(sum(np.array(input_df)))\n",
    "    return accuracy, get_severity(input_df, weights_df)\n",
    "\n",
    "def flatten(df):\n",
    "    return df.to_numpy().flatten()\n",
    "\n",
    "def counts_to_df(sampled_vals, num_labels):\n",
    "    mat = np.zeros((num_labels,num_labels), dtype=int)\n",
    "    for v in sampled_vals:\n",
    "        q, mod = divmod(v, num_labels)\n",
    "        mat[q][mod] += 1\n",
    "    return pd.DataFrame(mat)\n",
    "\n",
    "def bootstrap_sample(conf_mat, k):\n",
    "    num_labels = len(conf_mat.columns)\n",
    "    return counts_to_df(random.choices(population=range(num_labels**2), weights=flatten(conf_mat), k=k), num_labels)\n",
    "\n",
    "def bootstrap_ci(conf_mat, weights_df, iters, k, percentile=95):\n",
    "    iters, k = int(iters), int(k)\n",
    "    vals = []\n",
    "    for i in range(iters):\n",
    "        print()\n",
    "        vals.append(get_severity(bootstrap_sample(conf_mat, k), weights_df))\n",
    "    dif = (100 - percentile) / 2\n",
    "    return [np.percentile(vals, dif), np.percentile(vals, 100-dif)]\n",
    "\n",
    "with gr.Blocks() as demo:\n",
    "    with gr.Tab(\"Error severity matrix\"):\n",
    "        with gr.Row():\n",
    "            with gr.Column():            \n",
    "                sliders = []\n",
    "                for i in range(num_labels):\n",
    "                    for j in range(num_labels):\n",
    "                        if i != j:\n",
    "                            sliders.append(gr.Slider(1, 5, value=np.abs(i-j), step=1, label=\"Impact of misclassifying \"+ headers[j] + \" as \" + headers[i]))\n",
    "                submit_btn = gr.Button(\"Submit\")            \n",
    "            with gr.Column():\n",
    "                output_err_mat = gr.Dataframe(value = default_weights, datatype = \"number\", row_count = (num_labels, \"fixed\"), col_count=(num_labels,\"fixed\"), label=\"Error Severity Matrix\", interactive=0, headers=headers)\n",
    "            submit_btn.click(submit_vals, inputs=sliders, outputs=output_err_mat)\n",
    "    with gr.Tab(\"Calculate accuracy and Error Severity\"):\n",
    "        with gr.Row():\n",
    "            with gr.Column():\n",
    "                conf_df = gr.Dataframe(datatype = \"number\", row_count = (num_labels, \"fixed\"), col_count=(num_labels,\"fixed\"), label=\"Confusion Matrix\", interactive=1, headers=headers)\n",
    "                submit_btn = gr.Button(\"Submit\")\n",
    "                examples = gr.Examples(examples=example_conf_mats, inputs=[conf_df])\n",
    "            with gr.Column():\n",
    "                outputs = [gr.Textbox(label=\"Accuracy\"), gr.Textbox(label=\"Error Severity\")]\n",
    "            submit_btn.click(fn=get_acc, inputs=[conf_df,output_err_mat], outputs=outputs)\n",
    "        with gr.Row():\n",
    "            \n",
    "            with gr.Column():\n",
    "                gr.Markdown(\"Use bootstrapping to compute the 95% confidence interval for the Error Severity Index above\")\n",
    "                ci_inputs = [gr.Textbox(label=\"Iterations\", value=100), gr.Textbox(label=\"Sample size\", value=300)]    \n",
    "                submit_btn2 = gr.Button(\"Submit\")\n",
    "            with gr.Column():\n",
    "                output_ci = gr.Textbox(label=\"95% Confidence Interval\")\n",
    "            submit_btn2.click(fn=bootstrap_ci, inputs=[conf_df,output_err_mat, ci_inputs[0], ci_inputs[1]], outputs = output_ci)\n",
    "demo.launch()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PH",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "95392aae2145c7ea5cbfa9fd773fceb6d36e627f74f7ecde1b8de6e9d334481b"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}