asad

.gitattributes

@@ -38,15 +38,3 @@ ratio_proportion_change3/output/FS/bert_fine_tuned.model.ep32 filter=lfs diff=lf
 ratio_proportion_change3/output/IS/bert_fine_tuned.model.ep14 filter=lfs diff=lfs merge=lfs -text
 ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/highGRschool10/test_info.txt filter=lfs diff=lfs merge=lfs -text
 ratio_proportion_change3_2223/sch_largest_100-coded/output/highGRschool10/bert_fine_tuned.model.ep42 filter=lfs diff=lfs merge=lfs -text
-train_info.txt filter=lfs diff=lfs merge=lfs -text
-ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/lowGRschoolAll/test_info.txt filter=lfs diff=lfs merge=lfs -text
-ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/lowGRschoolAll/test.txt filter=lfs diff=lfs merge=lfs -text
-fullTest/test_info.txt filter=lfs diff=lfs merge=lfs -text
-fullTest/test.txt filter=lfs diff=lfs merge=lfs -text
-ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/test_info.txt filter=lfs diff=lfs merge=lfs -text
-ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/test.txt filter=lfs diff=lfs merge=lfs -text
-ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/fullTest/test_info.txt filter=lfs diff=lfs merge=lfs -text
-ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/fullTest/test.txt filter=lfs diff=lfs merge=lfs -text
-ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/fullTest/highGRschool10_/test_info.txt filter=lfs diff=lfs merge=lfs -text
-ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/fullTest/highGRschool10_/test.txt filter=lfs diff=lfs merge=lfs -text
-ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/fullTest/test_BKT.txt filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,5 +1,2 @@
 train_info.txt
-train.txt
-train_label.txt
 ratio_proportion_change3_2223/sch_largest_100-coded/logs/
-ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/

.ipynb_checkpoints/distinguish_high_low_label-checkpoint.ipynb

@@ -1,447 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "960bac80-51c7-4e9f-ad2d-84cd6c710f98",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import pickle\n",
-    "import pandas as pd"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "id": "a34f21d0-0854-4a54-8f93-67718b2f969e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "file_path = \"roc_data2.pkl\"\n",
-    "\n",
-    "# Open and load the pickle file\n",
-    "with open(file_path, 'rb') as file:\n",
-    "    data = pickle.load(file)\n",
-    "\n",
-    "\n",
-    "# Print or use the data\n",
-    "# data[2]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "f9febed4-ce50-4e30-96ea-4b538ce2f9a1",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "inc_slider=1\n",
-    "parent_location=\"ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/\"\n",
-    "test_info_location=parent_location+\"fullTest/test_info.txt\"\n",
-    "test_location=parent_location+\"fullTest/test.txt\"\n",
-    "test_info = pd.read_csv(test_info_location, sep=',', header=None, engine='python')\n",
-    "grad_rate_data = pd.DataFrame(pd.read_pickle('school_grduation_rate.pkl'),columns=['school_number','grad_rate'])  # Load the grad_rate data\n",
-    "\n",
-    "# Step 1: Extract unique school numbers from test_info\n",
-    "unique_schools = test_info[0].unique()\n",
-    "\n",
-    "# Step 2: Filter the grad_rate_data using the unique school numbers\n",
-    "schools = grad_rate_data[grad_rate_data['school_number'].isin(unique_schools)]\n",
-    "\n",
-    "# Define a threshold for high and low graduation rates (adjust as needed)\n",
-    "grad_rate_threshold = 0.9  \n",
-    "\n",
-    "# Step 4: Divide schools into high and low graduation rate groups\n",
-    "high_grad_schools = schools[schools['grad_rate'] >= grad_rate_threshold]['school_number'].unique()\n",
-    "low_grad_schools = schools[schools['grad_rate'] < grad_rate_threshold]['school_number'].unique()\n",
-    "\n",
-    "# Step 5: Sample percentage of schools from each group\n",
-    "high_sample = pd.Series(high_grad_schools).sample(frac=inc_slider/100, random_state=1).tolist()\n",
-    "low_sample = pd.Series(low_grad_schools).sample(frac=inc_slider/100, random_state=1).tolist()\n",
-    "\n",
-    "# Step 6: Combine the sampled schools\n",
-    "random_schools = high_sample + low_sample\n",
-    "\n",
-    "# Step 7: Get indices for the sampled schools\n",
-    "indices = test_info[test_info[0].isin(random_schools)].index.tolist()\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "id": "fdfdf4b6-2752-4a21-9880-869af69f20cf",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "high_indices = test_info[(test_info[0].isin(high_sample))].index.tolist()\n",
-    "low_indices = test_info[(test_info[0].isin(low_sample))].index.tolist()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "id": "a79a4598-5702-4cc8-9f07-8e18fdda648b",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "997"
-      ]
-     },
-     "execution_count": 7,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "len(high_indices)+len(low_indices)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "id": "4707f3e6-2f44-46d8-ad8c-b6c244f693af",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>0</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>5342</th>\n",
-       "      <td>PercentChange-0\\tNumeratorQuantity1-0\\tNumerat...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>5343</th>\n",
-       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>5344</th>\n",
-       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>5345</th>\n",
-       "      <td>PercentChange-0\\tNumeratorQuantity2-2\\tNumerat...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>5346</th>\n",
-       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tDenomin...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>113359</th>\n",
-       "      <td>PercentChange-0\\tNumeratorQuantity2-2\\tNumerat...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>113360</th>\n",
-       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>113361</th>\n",
-       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>113362</th>\n",
-       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>113363</th>\n",
-       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>997 rows × 1 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                                                        0\n",
-       "5342    PercentChange-0\\tNumeratorQuantity1-0\\tNumerat...\n",
-       "5343    PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
-       "5344    PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
-       "5345    PercentChange-0\\tNumeratorQuantity2-2\\tNumerat...\n",
-       "5346    PercentChange-0\\tNumeratorQuantity2-0\\tDenomin...\n",
-       "...                                                   ...\n",
-       "113359  PercentChange-0\\tNumeratorQuantity2-2\\tNumerat...\n",
-       "113360  PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
-       "113361  PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
-       "113362  PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
-       "113363  PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
-       "\n",
-       "[997 rows x 1 columns]"
-      ]
-     },
-     "execution_count": 8,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "# Load the test file and select rows based on indices\n",
-    "test = pd.read_csv(test_location, sep=',', header=None, engine='python')\n",
-    "selected_rows_df2 = test.loc[indices]\n",
-    "selected_rows_df2"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "id": "1d0c3d49-061f-486b-9c19-cf20945f3207",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "graduation_groups = [\n",
-    "    'high' if idx in high_indices else 'low' for idx in selected_rows_df2.index\n",
-    "]\n",
-    "# graduation_groups"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 43,
-   "id": "ad0ce4a1-27fa-4867-8061-4054dbb340df",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "t_label=data[0]\n",
-    "p_label=data[1]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 47,
-   "id": "a4f4a2b9-3134-42ac-871b-4e117098cd0e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Step 1: Align graduation_group, t_label, and p_label\n",
-    "aligned_labels = list(zip(graduation_groups, t_label, p_label))\n",
-    "\n",
-    "# Step 2: Separate the labels for high and low groups\n",
-    "high_t_labels = [t for grad, t, p in aligned_labels if grad == 'high']\n",
-    "low_t_labels = [t for grad, t, p in aligned_labels if grad == 'low']\n",
-    "\n",
-    "high_p_labels = [p for grad, t, p in aligned_labels if grad == 'high']\n",
-    "low_p_labels = [p for grad, t, p in aligned_labels if grad == 'low']\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 50,
-   "id": "c8e34660-83d0-46a1-a218-95d609e11729",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "997"
-      ]
-     },
-     "execution_count": 50,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "len(low_t_labels)+len(high_t_labels)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 51,
-   "id": "c11050db-2636-4c50-9cd4-b9943e5cee83",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, roc_curve, roc_auc_score"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 52,
-   "id": "e1309e93-7063-4f48-bbc7-11a0d449c34e",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "ROC-AUC Score for High Graduation Rate Group: 0.675\n",
-      "ROC-AUC Score for Low Graduation Rate Group: 0.7489795918367347\n"
-     ]
-    }
-   ],
-   "source": [
-    "high_roc_auc = roc_auc_score(high_t_labels, high_p_labels) if len(set(high_t_labels)) > 1 else None\n",
-    "low_roc_auc = roc_auc_score(low_t_labels, low_p_labels) if len(set(low_t_labels)) > 1 else None\n",
-    "\n",
-    "print(\"ROC-AUC Score for High Graduation Rate Group:\", high_roc_auc)\n",
-    "print(\"ROC-AUC Score for Low Graduation Rate Group:\", low_roc_auc)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "id": "a99e7812-817d-4f9f-b6fa-1a58aa3a34dc",
-   "metadata": {},
-   "outputs": [
-    {
-     "ename": "TypeError",
-     "evalue": "cannot convert the series to <class 'int'>",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
-      "Cell \u001b[1;32mIn[4], line 47\u001b[0m\n\u001b[0;32m     44\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(test_info_location, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m file:\n\u001b[0;32m     45\u001b[0m     data \u001b[38;5;241m=\u001b[39m file\u001b[38;5;241m.\u001b[39mreadlines()\n\u001b[1;32m---> 47\u001b[0m ideal_opt_task \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mint\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mtest_info\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m7\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# Assuming test_info[7] is accessible and holds the ideal task (1 or 2)\u001b[39;00m\n\u001b[0;32m     49\u001b[0m \u001b[38;5;66;03m# Initialize counters\u001b[39;00m\n\u001b[0;32m     50\u001b[0m task_counts \u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m     51\u001b[0m     \u001b[38;5;241m1\u001b[39m: {\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124monly_opt1\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m0\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124monly_opt2\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m0\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mboth\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m0\u001b[39m},\n\u001b[0;32m     52\u001b[0m     \u001b[38;5;241m2\u001b[39m: {\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124monly_opt1\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m0\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124monly_opt2\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m0\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mboth\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m0\u001b[39m}\n\u001b[0;32m     53\u001b[0m }\n",
-      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\site-packages\\pandas\\core\\series.py:230\u001b[0m, in \u001b[0;36m_coerce_method.<locals>.wrapper\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    222\u001b[0m     warnings\u001b[38;5;241m.\u001b[39mwarn(\n\u001b[0;32m    223\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCalling \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mconverter\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m on a single element Series is \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    224\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdeprecated and will raise a TypeError in the future. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    227\u001b[0m         stacklevel\u001b[38;5;241m=\u001b[39mfind_stack_level(),\n\u001b[0;32m    228\u001b[0m     )\n\u001b[0;32m    229\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m converter(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39miloc[\u001b[38;5;241m0\u001b[39m])\n\u001b[1;32m--> 230\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcannot convert the series to \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mconverter\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
-      "\u001b[1;31mTypeError\u001b[0m: cannot convert the series to <class 'int'>"
-     ]
-    }
-   ],
-   "source": [
-    "parent_location=\"ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/\"\n",
-    "test_info_location=parent_location+\"fullTest/test_info.txt\"\n",
-    "test_location=parent_location+\"fullTest/test.txt\"\n",
-    "test_info = pd.read_csv(test_info_location, sep=',', header=None, engine='python')\n",
-    "\n",
-    "def analyze_row(row, ideal_opt_task):\n",
-    "    # Split the row into fields\n",
-    "    fields = row.split(\"\\t\")\n",
-    "\n",
-    "    # Define tasks for OptionalTask_1, OptionalTask_2, and FinalAnswer\n",
-    "    optional_task_1_subtasks = [\"DenominatorFactor\", \"NumeratorFactor\", \"EquationAnswer\"]\n",
-    "    optional_task_2_subtasks = [\n",
-    "        \"FirstRow2:1\", \"FirstRow2:2\", \"FirstRow1:1\", \"FirstRow1:2\", \n",
-    "        \"SecondRow\", \"ThirdRow\"\n",
-    "    ]\n",
-    "    final_answer_tasks = [\"FinalAnswer\"]\n",
-    "\n",
-    "    # Helper function to evaluate task attempts\n",
-    "    def evaluate_tasks(fields, tasks):\n",
-    "        task_status = {}\n",
-    "        for task in tasks:\n",
-    "            relevant_attempts = [f for f in fields if task in f]\n",
-    "            if any(\"OK\" in attempt for attempt in relevant_attempts):\n",
-    "                task_status[task] = \"Attempted (Successful)\"\n",
-    "            elif any(\"ERROR\" in attempt for attempt in relevant_attempts):\n",
-    "                task_status[task] = \"Attempted (Error)\"\n",
-    "            elif any(\"JIT\" in attempt for attempt in relevant_attempts):\n",
-    "                task_status[task] = \"Attempted (JIT)\"\n",
-    "            else:\n",
-    "                task_status[task] = \"Unattempted\"\n",
-    "        return task_status\n",
-    "\n",
-    "    # Evaluate tasks for each category\n",
-    "    optional_task_1_status = evaluate_tasks(fields, optional_task_1_subtasks)\n",
-    "    optional_task_2_status = evaluate_tasks(fields, optional_task_2_subtasks)\n",
-    "\n",
-    "    # Check if tasks have any successful attempt\n",
-    "    opt1_done = any(status == \"Attempted (Successful)\" for status in optional_task_1_status.values())\n",
-    "    opt2_done = any(status == \"Attempted (Successful)\" for status in optional_task_2_status.values())\n",
-    "\n",
-    "    return opt1_done, opt2_done\n",
-    "\n",
-    "# Read data from test_info.txt\n",
-    "with open(test_info_location, \"r\") as file:\n",
-    "    data = file.readlines()\n",
-    "\n",
-    "ideal_opt_task = int(test_info[6])  # Assuming test_info[7] is accessible and holds the ideal task (1 or 2)\n",
-    "\n",
-    "# Initialize counters\n",
-    "task_counts = {\n",
-    "    1: {\"only_opt1\": 0, \"only_opt2\": 0, \"both\": 0},\n",
-    "    2: {\"only_opt1\": 0, \"only_opt2\": 0, \"both\": 0}\n",
-    "}\n",
-    "\n",
-    "for row in data:\n",
-    "    row = row.strip()\n",
-    "    if not row:\n",
-    "        continue\n",
-    "    opt1_done, opt2_done = analyze_row(row, ideal_opt_task)\n",
-    "\n",
-    "    if ideal_opt_task == 0:\n",
-    "        if opt1_done and not opt2_done:\n",
-    "            task_counts[1][\"only_opt1\"] += 1\n",
-    "        elif not opt1_done and opt2_done:\n",
-    "            task_counts[1][\"only_opt2\"] += 1\n",
-    "        elif opt1_done and opt2_done:\n",
-    "            task_counts[1][\"both\"] += 1\n",
-    "    elif ideal_opt_task == 1:\n",
-    "        if opt1_done and not opt2_done:\n",
-    "            task_counts[2][\"only_opt1\"] += 1\n",
-    "        elif not opt1_done and opt2_done:\n",
-    "            task_counts[2][\"only_opt2\"] += 1\n",
-    "        elif opt1_done and opt2_done:\n",
-    "            task_counts[2][\"both\"] += 1\n",
-    "\n",
-    "# Create a string output for results\n",
-    "output_summary = \"Task Analysis Summary:\\n\"\n",
-    "output_summary += \"-----------------------\\n\"\n",
-    "\n",
-    "for ideal_task, counts in task_counts.items():\n",
-    "    output_summary += f\"Ideal Task = OptionalTask_{ideal_task}:\\n\"\n",
-    "    output_summary += f\"  Only OptionalTask_1 done: {counts['only_opt1']}\\n\"\n",
-    "    output_summary += f\"  Only OptionalTask_2 done: {counts['only_opt2']}\\n\"\n",
-    "    output_summary += f\"  Both done: {counts['both']}\\n\"\n",
-    "\n",
-    "print(output_summary)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "65ad9383-741f-44eb-8e8f-853ee7bc52a2",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.12.4"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

Untitled.ipynb

@@ -623,7 +623,7 @@
    "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/base-cu113:m122"
   },
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "Python 3",
    "language": "python",
    "name": "python3"
   },
@@ -637,7 +637,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.4"
+   "version": "3.10.14"
   }
  },
  "nbformat": 4,

app.py

@@ -8,41 +8,24 @@ import shutil
 import matplotlib.pyplot as plt
 from sklearn.metrics import roc_curve, auc
 import pandas as pd
-from sklearn.metrics import roc_auc_score
 # Define the function to process the input file and model selection
 
-def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
+def process_file(file,label,info,model_name,inc_slider,progress=Progress(track_tqdm=True)):
     # progress = gr.Progress(track_tqdm=True)
-
     progress(0, desc="Starting the processing") 
-    # with open(file.name, 'r') as f:
-    #     content = f.read()
-    # saved_test_dataset = "train.txt"
-    # saved_test_label = "train_label.txt"
-    # saved_train_info="train_info.txt"
+    with open(file.name, 'r') as f:
+        content = f.read()
+    saved_test_dataset = "train.txt"
+    saved_test_label = "train_label.txt"
+    saved_train_info="train_info.txt"
     # Save the uploaded file content to a specified location
-    # shutil.copyfile(file.name, saved_test_dataset)
-    # shutil.copyfile(label.name, saved_test_label)
-    # shutil.copyfile(info.name, saved_train_info)
-    parent_location="ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/"
-    test_info_location=parent_location+"fullTest/test_info.txt"
-    test_location=parent_location+"fullTest/test.txt"
-    if(model_name=="ASTRA-FT-HGR"):
-        finetune_task="highGRschool10"
-        # test_info_location=parent_location+"fullTest/test_info.txt"
-        # test_location=parent_location+"fullTest/test.txt"
-    elif(model_name== "ASTRA-FT-LGR" ):
-        finetune_task="lowGRschoolAll"
-        # test_info_location=parent_location+"lowGRschoolAll/test_info.txt"
-        # test_location=parent_location+"lowGRschoolAll/test.txt"
-    elif(model_name=="ASTRA-FT-FULL"):
-        # test_info_location=parent_location+"fullTest/test_info.txt"
-        # test_location=parent_location+"fullTest/test.txt"
-        finetune_task="fullTest"
-    else:
-        finetune_task=None
+    shutil.copyfile(file.name, saved_test_dataset)
+    shutil.copyfile(label.name, saved_test_label)
+    shutil.copyfile(info.name, saved_train_info)
+
+
     # Load the test_info file and the graduation rate file
-    test_info = pd.read_csv(test_info_location, sep=',', header=None, engine='python')
+    test_info = pd.read_csv('train_info.txt', sep=',', header=None, engine='python')
     grad_rate_data = pd.DataFrame(pd.read_pickle('school_grduation_rate.pkl'),columns=['school_number','grad_rate'])  # Load the grad_rate data
 
     # Step 1: Extract unique school numbers from test_info
@@ -67,39 +50,24 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
 
     # Step 7: Get indices for the sampled schools
     indices = test_info[test_info[0].isin(random_schools)].index.tolist()
-    high_indices = test_info[(test_info[0].isin(high_sample))].index.tolist()
-    low_indices = test_info[(test_info[0].isin(low_sample))].index.tolist()
 
     # Load the test file and select rows based on indices
-    test = pd.read_csv(test_location, sep=',', header=None, engine='python')
+    test = pd.read_csv('train.txt', sep=',', header=None, engine='python')
     selected_rows_df2 = test.loc[indices]
 
     # Save the selected rows to a file
     selected_rows_df2.to_csv('selected_rows.txt', sep='\t', index=False, header=False, quoting=3, escapechar=' ')
 
-    graduation_groups = [
-    'high' if idx in high_indices else 'low' for idx in selected_rows_df2.index
-    ]
-
-    
-    with open("roc_data2.pkl", 'rb') as file:
-        data = pickle.load(file)
-    t_label=data[0]
-    p_label=data[1]
-    # Step 1: Align graduation_group, t_label, and p_label
-    aligned_labels = list(zip(graduation_groups, t_label, p_label))
-
-    # Step 2: Separate the labels for high and low groups
-    high_t_labels = [t for grad, t, p in aligned_labels if grad == 'high']
-    low_t_labels = [t for grad, t, p in aligned_labels if grad == 'low']
-
-    high_p_labels = [p for grad, t, p in aligned_labels if grad == 'high']
-    low_p_labels = [p for grad, t, p in aligned_labels if grad == 'low']
-
-    high_roc_auc = roc_auc_score(high_t_labels, high_p_labels) if len(set(high_t_labels)) > 1 else None
-    low_roc_auc = roc_auc_score(low_t_labels, low_p_labels) if len(set(low_t_labels)) > 1 else None
+   
     # For demonstration purposes, we'll just return the content with the selected model name
-
+    if(model_name=="High Graduated Schools"):
+        finetune_task="highGRschool10"
+    elif(model_name== "Low Graduated Schools" ):
+        finetune_task="highGRschool10"
+    elif(model_name=="Full Set"):
+        finetune_task="highGRschool10"
+    else:
+        finetune_task=None
     # print(checkpoint)
     progress(0.1, desc="Files created and saved")
     # if (inc_val<5):
@@ -108,99 +76,11 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
     #     model_name="highGRschool10"
     # else:
     #     model_name="highGRschool10"
-    # Function to analyze each row
-    def analyze_row(row):
-    # Split the row into fields
-        fields = row.split("\t")
-
-        # Define tasks for OptionalTask_1, OptionalTask_2, and FinalAnswer
-        optional_task_1_subtasks = ["DenominatorFactor", "NumeratorFactor", "EquationAnswer"]
-        optional_task_2_subtasks = [
-            "FirstRow2:1", "FirstRow2:2", "FirstRow1:1", "FirstRow1:2", 
-            "SecondRow", "ThirdRow"
-        ]
-
-        # Helper function to evaluate task attempts
-        def evaluate_tasks(fields, tasks):
-            task_status = {}
-            for task in tasks:
-                relevant_attempts = [f for f in fields if task in f]
-                if any("OK" in attempt for attempt in relevant_attempts):
-                    task_status[task] = "Attempted (Successful)"
-                elif any("ERROR" in attempt for attempt in relevant_attempts):
-                    task_status[task] = "Attempted (Error)"
-                elif any("JIT" in attempt for attempt in relevant_attempts):
-                    task_status[task] = "Attempted (JIT)"
-                else:
-                    task_status[task] = "Unattempted"
-            return task_status
-
-        # Evaluate tasks for each category
-        optional_task_1_status = evaluate_tasks(fields, optional_task_1_subtasks)
-        optional_task_2_status = evaluate_tasks(fields, optional_task_2_subtasks)
-
-        # Check if tasks have any successful attempt
-        opt1_done = any(status == "Attempted (Successful)" for status in optional_task_1_status.values())
-        opt2_done = any(status == "Attempted (Successful)" for status in optional_task_2_status.values())
-
-        return opt1_done, opt2_done
-
-    # Read data from test_info.txt
-    # Read data from test_info.txt
-    with open(test_info_location, "r") as file:
-        data = file.readlines()
-
-    # Assuming test_info[7] is a list with ideal tasks for each instance
-    ideal_tasks = test_info[6]  # A list where each element is either 1 or 2
-
-    # Initialize counters
-    task_counts = {
-        1: {"only_opt1": 0, "only_opt2": 0, "both": 0},
-        2: {"only_opt1": 0, "only_opt2": 0, "both": 0}
-    }
-
-    # Analyze rows
-    for i, row in enumerate(data):
-        row = row.strip()
-        if not row:
-            continue
-
-        ideal_task = ideal_tasks[i]  # Get the ideal task for the current row
-        opt1_done, opt2_done = analyze_row(row)
-
-        if ideal_task == 0:
-            if opt1_done and not opt2_done:
-                task_counts[1]["only_opt1"] += 1
-            elif not opt1_done and opt2_done:
-                task_counts[1]["only_opt2"] += 1
-            elif opt1_done and opt2_done:
-                task_counts[1]["both"] += 1
-        elif ideal_task == 1:
-            if opt1_done and not opt2_done:
-                task_counts[2]["only_opt1"] += 1
-            elif not opt1_done and opt2_done:
-                task_counts[2]["only_opt2"] += 1
-            elif opt1_done and opt2_done:
-                task_counts[2]["both"] += 1
-
-    # Create a string output for results
-    output_summary = "Task Analysis Summary:\n"
-    output_summary += "-----------------------\n"
-
-    for ideal_task, counts in task_counts.items():
-        output_summary += f"Ideal Task = OptionalTask_{ideal_task}:\n"
-        output_summary += f"  Only OptionalTask_1 done: {counts['only_opt1']}\n"
-        output_summary += f"  Only OptionalTask_2 done: {counts['only_opt2']}\n"
-        output_summary += f"  Both done: {counts['both']}\n"
-
-    # print(output_summary)
-
-    progress(0.2, desc="analysis done!! Executing models")
-    print("finetuned task: ",finetune_task)
+    progress(0.2, desc="Executing models")
     subprocess.run([
         "python", "new_test_saved_finetuned_model.py",
         "-workspace_name", "ratio_proportion_change3_2223/sch_largest_100-coded",
-        "-finetune_task", finetune_task,
+        "-finetune_task", "highGRschool10",
         "-test_dataset_path","../../../../selected_rows.txt",
         # "-test_label_path","../../../../train_label.txt",
         "-finetuned_bert_classifier_checkpoint", 
@@ -218,17 +98,15 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
                 result[key]=value
             else:
                  result[key]=float(value)
-    result["ROC score of HGR"]=high_roc_auc
-    result["ROC score of LGR"]=low_roc_auc
 # Create a plot
     with open("roc_data.pkl", "rb") as f:
         fpr, tpr, _ = pickle.load(f)
-    # print(fpr,tpr)
+
     roc_auc = auc(fpr, tpr)
     fig, ax = plt.subplots()
     ax.plot(fpr, tpr, color='blue', lw=2, label=f'ROC curve (area = {roc_auc:.2f})')
     ax.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
-    ax.set(xlabel='False Positive Rate', ylabel='True Positive Rate', title=f'(Receiver Operating Curve) ROC')
+    ax.set(xlabel='False Positive Rate', ylabel='True Positive Rate', title=f'ROC Curve: {model_name}')
     ax.legend(loc="lower right")
     ax.grid()
 
@@ -236,75 +114,30 @@ def process_file(model_name,inc_slider,progress=Progress(track_tqdm=True)):
     plot_path = "plot.png"
     fig.savefig(plot_path)
     plt.close(fig)
-
-
-    
-
     progress(1.0)
     # Prepare text output
     text_output = f"Model: {model_name}\nResult:\n{result}"
     # Prepare text output with HTML formatting
     text_output = f"""
     Model: {model_name}\n
+    Result Summary:\n
     -----------------\n
-
+    Precision: {result['precisions']:.2f}\n
+    Recall: {result['recalls']:.2f}\n
     Time Taken: {result['time_taken_from_start']:.2f} seconds\n
     Total Schools in test: {len(unique_schools):.4f}\n
-    Total number of instances having Schools with HGR : {len(high_sample):.4f}\n
-    Total number of instances having Schools with LGR: {len(low_sample):.4f}\n
-
-    ROC score of HGR: {high_roc_auc}\n
-    ROC score of LGR: {low_roc_auc}\n
+    Total Schools taken: {len(random_schools):.4f}\n
+    High grad schools: {len(high_sample):.4f}\n
+    Low grad schools: {len(low_sample):.4f}\n
     -----------------\n
+    Note: The ROC Curve is also displayed for the evaluation.
     """
-    return text_output,plot_path,output_summary
+    return text_output,plot_path
 
 # List of models for the dropdown menu
 
-# models = ["ASTRA-FT-HGR", "ASTRA-FT-LGR", "ASTRA-FT-FULL"]
-models = ["ASTRA-FT-HGR", "ASTRA-FT-FULL"]
-content = """
-<h1 style="color: white;">ASTRA: An AI Model for Analyzing Math Strategies</h1>
-
-<h3 style="color: white;">
-    <a href="https://drive.google.com/file/d/1lbEpg8Se1ugTtkjreD8eXIg7qrplhWan/view" style="color: #1E90FF; text-decoration: none;">Link To Paper</a> | 
-    <a href="https://github.com/Syudu41/ASTRA---Gates-Project" style="color: #1E90FF; text-decoration: none;">GitHub</a> | 
-    <a href="#" style="color: #1E90FF; text-decoration: none;">Project Page</a>
-</h3>
-
-<p style="color: white;">Welcome to a demo of ASTRA. ASTRA is a collaborative research project between researchers at the 
-<a href="https://www.memphis.edu" style="color: #1E90FF; text-decoration: none;">University of Memphis</a> and 
-<a href="https://www.carnegielearning.com" style="color: #1E90FF; text-decoration: none;">Carnegie Learning</a> 
-to utilize AI to improve our understanding of math learning strategies.</p>
-
-<p style="color: white;">This demo has been developed with a pre-trained model (based on an architecture similar to BERT) 
-that learns math strategies using data collected from hundreds of schools in the U.S. who have used 
-Carnegie Learning's MATHia (formerly known as Cognitive Tutor), the flagship Intelligent Tutor 
-that is part of a core, blended math curriculum.</p>
-
-<p style="color: white;">For this demo, we have used data from a specific domain (teaching ratio and proportions) within 
-7th grade math. The fine-tuning based on the pre-trained models learns to predict which strategies 
-lead to correct vs. incorrect solutions.</p>
+models = ["High Graduated Schools", "Low Graduated Schools", "Full Set"]
 
-<p style="color: white;">To use the demo, please follow these steps:</p>
-
-<ol style="color: white;">
-    <li style="color: white;">Select a fine-tuned model:
-        <ul style="color: white;">
-            <li style="color: white;">ASTRA-FT-HGR: Fine-tuned with a small sample of data from schools that have a high graduation rate.</li>
-            <li style="color: white;">ASTRA-FT-Full: Fine-tuned with a small sample of data from a mix of schools that have high/low graduation rates.</li>
-        </ul>
-    </li>
-    <li style="color: white;">Select a percentage of schools to analyze (selecting a large percentage may take a long time).</li>
-    <li style="color: white;">View Results:
-        <ul>
-            <li style="color: white;">The results from the fine-tuned model are displayed on the dashboard.</li>
-            <li style="color: white;">The results are shown separately for schools that have high and low graduation rates.</li>
-        </ul>
-    </li>
-</ol>
-"""
-# CSS styling for white text
 # Create the Gradio interface
 with gr.Blocks(css="""
     body {
@@ -312,7 +145,6 @@ with gr.Blocks(css="""
         font-family: 'Arial', sans-serif;
         color: #f5f5f5!important;;
     }
-
     .gradio-container {
         max-width: 850px!important;
         margin: 0 auto!important;;
@@ -484,35 +316,31 @@ tbody.svelte-18wv37q>tr.svelte-18wv37q:nth-child(odd) {
     color: white;
     background: #aca7b2;
 }
-
 .gradio-container-4-31-4 .prose h1, .gradio-container-4-31-4 .prose h2, .gradio-container-4-31-4 .prose h3, .gradio-container-4-31-4 .prose h4, .gradio-container-4-31-4 .prose h5 {
 
     color: white;
-}
 """) as demo:
-    
     gr.Markdown("<h1 id='title'>ASTRA</h1>", elem_id="title")
-    gr.Markdown(content)
+    gr.Markdown("<p class='description'>Upload a .txt file and select a model from the dropdown menu.</p>")
     
     with gr.Row():
-        # file_input = gr.File(label="Upload a test file", file_types=['.txt'], elem_classes="file-box")
-        # label_input = gr.File(label="Upload test labels", file_types=['.txt'], elem_classes="file-box")
+        file_input = gr.File(label="Upload a test file", file_types=['.txt'], elem_classes="file-box")
+        label_input = gr.File(label="Upload test labels", file_types=['.txt'], elem_classes="file-box")
 
-        # info_input = gr.File(label="Upload test info", file_types=['.txt'], elem_classes="file-box")
+        info_input = gr.File(label="Upload test info", file_types=['.txt'], elem_classes="file-box")
     
-        model_dropdown = gr.Dropdown(choices=models, label="Select Fine-tuned Model", elem_classes="dropdown-menu")
+    model_dropdown = gr.Dropdown(choices=models, label="Select Finetune Task", elem_classes="dropdown-menu")
 
     
     increment_slider = gr.Slider(minimum=1, maximum=100, step=1, label="Schools Percentage", value=1)
-    gr.Markdown("<p class='description'>Dashboard</p>")
+    
     with gr.Row():
-        output_text = gr.Textbox(label="")
-        output_image = gr.Image(label="ROC")
-        output_summary = gr.Textbox(label="Summary")
+        output_text = gr.Textbox(label="Output Text")
+        output_image = gr.Image(label="Output Plot")
 
     btn = gr.Button("Submit")
  
-    btn.click(fn=process_file, inputs=[model_dropdown,increment_slider], outputs=[output_text,output_image,output_summary])
+    btn.click(fn=process_file, inputs=[file_input,label_input,info_input,model_dropdown,increment_slider], outputs=[output_text,output_image])
 
 
 # Launch the app

distinguish_high_low_label.ipynb

@@ -1,451 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "960bac80-51c7-4e9f-ad2d-84cd6c710f98",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import pickle\n",
-    "import pandas as pd"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "id": "a34f21d0-0854-4a54-8f93-67718b2f969e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "file_path = \"roc_data2.pkl\"\n",
-    "\n",
-    "# Open and load the pickle file\n",
-    "with open(file_path, 'rb') as file:\n",
-    "    data = pickle.load(file)\n",
-    "\n",
-    "\n",
-    "# Print or use the data\n",
-    "# data[2]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "f9febed4-ce50-4e30-96ea-4b538ce2f9a1",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "inc_slider=1\n",
-    "parent_location=\"ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/\"\n",
-    "test_info_location=parent_location+\"fullTest/test_info.txt\"\n",
-    "test_location=parent_location+\"fullTest/test.txt\"\n",
-    "test_info = pd.read_csv(test_info_location, sep=',', header=None, engine='python')\n",
-    "grad_rate_data = pd.DataFrame(pd.read_pickle('school_grduation_rate.pkl'),columns=['school_number','grad_rate'])  # Load the grad_rate data\n",
-    "\n",
-    "# Step 1: Extract unique school numbers from test_info\n",
-    "unique_schools = test_info[0].unique()\n",
-    "\n",
-    "# Step 2: Filter the grad_rate_data using the unique school numbers\n",
-    "schools = grad_rate_data[grad_rate_data['school_number'].isin(unique_schools)]\n",
-    "\n",
-    "# Define a threshold for high and low graduation rates (adjust as needed)\n",
-    "grad_rate_threshold = 0.9  \n",
-    "\n",
-    "# Step 4: Divide schools into high and low graduation rate groups\n",
-    "high_grad_schools = schools[schools['grad_rate'] >= grad_rate_threshold]['school_number'].unique()\n",
-    "low_grad_schools = schools[schools['grad_rate'] < grad_rate_threshold]['school_number'].unique()\n",
-    "\n",
-    "# Step 5: Sample percentage of schools from each group\n",
-    "high_sample = pd.Series(high_grad_schools).sample(frac=inc_slider/100, random_state=1).tolist()\n",
-    "low_sample = pd.Series(low_grad_schools).sample(frac=inc_slider/100, random_state=1).tolist()\n",
-    "\n",
-    "# Step 6: Combine the sampled schools\n",
-    "random_schools = high_sample + low_sample\n",
-    "\n",
-    "# Step 7: Get indices for the sampled schools\n",
-    "indices = test_info[test_info[0].isin(random_schools)].index.tolist()\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "id": "fdfdf4b6-2752-4a21-9880-869af69f20cf",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "high_indices = test_info[(test_info[0].isin(high_sample))].index.tolist()\n",
-    "low_indices = test_info[(test_info[0].isin(low_sample))].index.tolist()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "id": "a79a4598-5702-4cc8-9f07-8e18fdda648b",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "997"
-      ]
-     },
-     "execution_count": 7,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "len(high_indices)+len(low_indices)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "id": "4707f3e6-2f44-46d8-ad8c-b6c244f693af",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>0</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>5342</th>\n",
-       "      <td>PercentChange-0\\tNumeratorQuantity1-0\\tNumerat...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>5343</th>\n",
-       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>5344</th>\n",
-       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>5345</th>\n",
-       "      <td>PercentChange-0\\tNumeratorQuantity2-2\\tNumerat...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>5346</th>\n",
-       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tDenomin...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>113359</th>\n",
-       "      <td>PercentChange-0\\tNumeratorQuantity2-2\\tNumerat...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>113360</th>\n",
-       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>113361</th>\n",
-       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>113362</th>\n",
-       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>113363</th>\n",
-       "      <td>PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>997 rows × 1 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                                                        0\n",
-       "5342    PercentChange-0\\tNumeratorQuantity1-0\\tNumerat...\n",
-       "5343    PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
-       "5344    PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
-       "5345    PercentChange-0\\tNumeratorQuantity2-2\\tNumerat...\n",
-       "5346    PercentChange-0\\tNumeratorQuantity2-0\\tDenomin...\n",
-       "...                                                   ...\n",
-       "113359  PercentChange-0\\tNumeratorQuantity2-2\\tNumerat...\n",
-       "113360  PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
-       "113361  PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
-       "113362  PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
-       "113363  PercentChange-0\\tNumeratorQuantity2-0\\tNumerat...\n",
-       "\n",
-       "[997 rows x 1 columns]"
-      ]
-     },
-     "execution_count": 8,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "# Load the test file and select rows based on indices\n",
-    "test = pd.read_csv(test_location, sep=',', header=None, engine='python')\n",
-    "selected_rows_df2 = test.loc[indices]\n",
-    "selected_rows_df2"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "id": "1d0c3d49-061f-486b-9c19-cf20945f3207",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "graduation_groups = [\n",
-    "    'high' if idx in high_indices else 'low' for idx in selected_rows_df2.index\n",
-    "]\n",
-    "# graduation_groups"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 43,
-   "id": "ad0ce4a1-27fa-4867-8061-4054dbb340df",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "t_label=data[0]\n",
-    "p_label=data[1]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 47,
-   "id": "a4f4a2b9-3134-42ac-871b-4e117098cd0e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Step 1: Align graduation_group, t_label, and p_label\n",
-    "aligned_labels = list(zip(graduation_groups, t_label, p_label))\n",
-    "\n",
-    "# Step 2: Separate the labels for high and low groups\n",
-    "high_t_labels = [t for grad, t, p in aligned_labels if grad == 'high']\n",
-    "low_t_labels = [t for grad, t, p in aligned_labels if grad == 'low']\n",
-    "\n",
-    "high_p_labels = [p for grad, t, p in aligned_labels if grad == 'high']\n",
-    "low_p_labels = [p for grad, t, p in aligned_labels if grad == 'low']\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 50,
-   "id": "c8e34660-83d0-46a1-a218-95d609e11729",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "997"
-      ]
-     },
-     "execution_count": 50,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "len(low_t_labels)+len(high_t_labels)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 51,
-   "id": "c11050db-2636-4c50-9cd4-b9943e5cee83",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, roc_curve, roc_auc_score"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 52,
-   "id": "e1309e93-7063-4f48-bbc7-11a0d449c34e",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "ROC-AUC Score for High Graduation Rate Group: 0.675\n",
-      "ROC-AUC Score for Low Graduation Rate Group: 0.7489795918367347\n"
-     ]
-    }
-   ],
-   "source": [
-    "high_roc_auc = roc_auc_score(high_t_labels, high_p_labels) if len(set(high_t_labels)) > 1 else None\n",
-    "low_roc_auc = roc_auc_score(low_t_labels, low_p_labels) if len(set(low_t_labels)) > 1 else None\n",
-    "\n",
-    "print(\"ROC-AUC Score for High Graduation Rate Group:\", high_roc_auc)\n",
-    "print(\"ROC-AUC Score for Low Graduation Rate Group:\", low_roc_auc)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "id": "a99e7812-817d-4f9f-b6fa-1a58aa3a34dc",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Task Analysis Summary:\n",
-      "-----------------------\n",
-      "Ideal Task = OptionalTask_1:\n",
-      "  Only OptionalTask_1 done: 22501\n",
-      "  Only OptionalTask_2 done: 20014\n",
-      "  Both done: 24854\n",
-      "Ideal Task = OptionalTask_2:\n",
-      "  Only OptionalTask_1 done: 12588\n",
-      "  Only OptionalTask_2 done: 18942\n",
-      "  Both done: 15147\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "def analyze_row(row):\n",
-    "    # Split the row into fields\n",
-    "    fields = row.split(\"\\t\")\n",
-    "\n",
-    "    # Define tasks for OptionalTask_1, OptionalTask_2, and FinalAnswer\n",
-    "    optional_task_1_subtasks = [\"DenominatorFactor\", \"NumeratorFactor\", \"EquationAnswer\"]\n",
-    "    optional_task_2_subtasks = [\n",
-    "        \"FirstRow2:1\", \"FirstRow2:2\", \"FirstRow1:1\", \"FirstRow1:2\", \n",
-    "        \"SecondRow\", \"ThirdRow\"\n",
-    "    ]\n",
-    "\n",
-    "    # Helper function to evaluate task attempts\n",
-    "    def evaluate_tasks(fields, tasks):\n",
-    "        task_status = {}\n",
-    "        for task in tasks:\n",
-    "            relevant_attempts = [f for f in fields if task in f]\n",
-    "            if any(\"OK\" in attempt for attempt in relevant_attempts):\n",
-    "                task_status[task] = \"Attempted (Successful)\"\n",
-    "            elif any(\"ERROR\" in attempt for attempt in relevant_attempts):\n",
-    "                task_status[task] = \"Attempted (Error)\"\n",
-    "            elif any(\"JIT\" in attempt for attempt in relevant_attempts):\n",
-    "                task_status[task] = \"Attempted (JIT)\"\n",
-    "            else:\n",
-    "                task_status[task] = \"Unattempted\"\n",
-    "        return task_status\n",
-    "\n",
-    "    # Evaluate tasks for each category\n",
-    "    optional_task_1_status = evaluate_tasks(fields, optional_task_1_subtasks)\n",
-    "    optional_task_2_status = evaluate_tasks(fields, optional_task_2_subtasks)\n",
-    "\n",
-    "    # Check if tasks have any successful attempt\n",
-    "    opt1_done = any(status == \"Attempted (Successful)\" for status in optional_task_1_status.values())\n",
-    "    opt2_done = any(status == \"Attempted (Successful)\" for status in optional_task_2_status.values())\n",
-    "\n",
-    "    return opt1_done, opt2_done\n",
-    "\n",
-    "# Read data from test_info.txt\n",
-    "# Read data from test_info.txt\n",
-    "with open(test_info_location, \"r\") as file:\n",
-    "    data = file.readlines()\n",
-    "\n",
-    "# Assuming test_info[7] is a list with ideal tasks for each instance\n",
-    "ideal_tasks = test_info[6]  # A list where each element is either 1 or 2\n",
-    "\n",
-    "# Initialize counters\n",
-    "task_counts = {\n",
-    "    1: {\"only_opt1\": 0, \"only_opt2\": 0, \"both\": 0},\n",
-    "    2: {\"only_opt1\": 0, \"only_opt2\": 0, \"both\": 0}\n",
-    "}\n",
-    "\n",
-    "# Analyze rows\n",
-    "for i, row in enumerate(data):\n",
-    "    row = row.strip()\n",
-    "    if not row:\n",
-    "        continue\n",
-    "\n",
-    "    ideal_task = ideal_tasks[i]  # Get the ideal task for the current row\n",
-    "    opt1_done, opt2_done = analyze_row(row)\n",
-    "\n",
-    "    if ideal_task == 0:\n",
-    "        if opt1_done and not opt2_done:\n",
-    "            task_counts[1][\"only_opt1\"] += 1\n",
-    "        elif not opt1_done and opt2_done:\n",
-    "            task_counts[1][\"only_opt2\"] += 1\n",
-    "        elif opt1_done and opt2_done:\n",
-    "            task_counts[1][\"both\"] += 1\n",
-    "    elif ideal_task == 1:\n",
-    "        if opt1_done and not opt2_done:\n",
-    "            task_counts[2][\"only_opt1\"] += 1\n",
-    "        elif not opt1_done and opt2_done:\n",
-    "            task_counts[2][\"only_opt2\"] += 1\n",
-    "        elif opt1_done and opt2_done:\n",
-    "            task_counts[2][\"both\"] += 1\n",
-    "\n",
-    "# Create a string output for results\n",
-    "output_summary = \"Task Analysis Summary:\\n\"\n",
-    "output_summary += \"-----------------------\\n\"\n",
-    "\n",
-    "for ideal_task, counts in task_counts.items():\n",
-    "    output_summary += f\"Ideal Task = OptionalTask_{ideal_task}:\\n\"\n",
-    "    output_summary += f\"  Only OptionalTask_1 done: {counts['only_opt1']}\\n\"\n",
-    "    output_summary += f\"  Only OptionalTask_2 done: {counts['only_opt2']}\\n\"\n",
-    "    output_summary += f\"  Both done: {counts['both']}\\n\"\n",
-    "\n",
-    "print(output_summary)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "65ad9383-741f-44eb-8e8f-853ee7bc52a2",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.12.4"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

fullTest/test.txt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2a479561b801a43249b6a8aceed5f32d16cec3d2f40956ed02640b6dcab0bdfe
-size 21353853

fullTest/test_info.txt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dbb182b48eecce59c4e61f82a23d8af2866d9327f0543aca3546880fdb0d6003
-size 166442240

new_test_saved_finetuned_model.py

@@ -221,12 +221,9 @@ class BERTFineTuneTrainer:
                 for key, value in final_msg.items():
                     file.write(f"{key}: {value}\n")
             print(final_msg)
-            # print(type(plabels),type(tlabels),plabels,tlabels) 
             fpr, tpr, thresholds = roc_curve(tlabels, positive_class_probs)
             with open("roc_data.pkl", "wb") as f:
                 pickle.dump((fpr, tpr, thresholds), f)
-            with open("roc_data2.pkl", "wb") as f:
-                pickle.dump((tlabels,positive_class_probs), f)                
             print(final_msg)
             f.close()
             with open(self.log_folder_path+f"/log_{phase}_finetuned_info.txt", 'a') as f1:
@@ -429,7 +426,6 @@ class BERTFineTuneCalibratedTrainer:
             auc_score = roc_auc_score(tlabels, positive_class_probs)
             end_time = time.time()
             final_msg = {
-                "this one":"this one",
                 "avg_loss": avg_loss / len(data_iter),
                 "total_acc": total_correct * 100.0 / total_element,
                 "precisions": precisions,
@@ -444,8 +440,7 @@ class BERTFineTuneCalibratedTrainer:
             with open("result.txt", 'w') as file:
                 for key, value in final_msg.items():
                     file.write(f"{key}: {value}\n")
-            with open("plabels.txt","w") as file:
-                file.write(plabels)          
+                    
             print(final_msg)
             fpr, tpr, thresholds = roc_curve(tlabels, positive_class_probs)
             f.close()

ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/fullTest/highGRschool10_/test.txt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:20a028aa7529f6c68f16ba09e038ef969ca61aa22ee1e41f5e0474883aabbddc
-size 24775790

ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/fullTest/highGRschool10_/test_info.txt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f29c4c3585b70ef5a1fc0c107d9d96c63b7adae0659789b90f5bfab97df57026
-size 123225375

ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/fullTest/test.txt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:35569d6f81ef85e6353f36912c1cb79bfb723fe7d2476e10afcb745c170c5130
-size 24672844

ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/fullTest/test_BKT.txt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dec171098a0b444d3b8a3de8497345e8806440038756ce51a575314e6c414647
-size 20086086

ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/fullTest/test_info.txt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a6aadba0002bfdfde835b8837b3ff36cd84c64c3e23b6589ec1d002b4b62c2f4
-size 122629427

ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/lowGRschoolAll/test.txt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e738c87fcbcc3e0362199ea2b7f9ef06093fb3f9e7a5f8c5ab828602e52230f9
-size 16005023

ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/lowGRschoolAll/test_info.txt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ef4862f5c282efdfa49e13ed0f6cb344abcb7ae07fdfba535d48193bb8a3c1ed
-size 81939614

ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/test.txt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2a479561b801a43249b6a8aceed5f32d16cec3d2f40956ed02640b6dcab0bdfe
-size 21353853

ratio_proportion_change3_2223/sch_largest_100-coded/finetuning/test_info.txt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dbb182b48eecce59c4e61f82a23d8af2866d9327f0543aca3546880fdb0d6003
-size 166442240

result.txt

@@ -1,7 +1,7 @@
-avg_loss: 0.5841353535652161
-total_acc: 69.00702106318957
-precisions: 0.7236623191454734
-recalls: 0.6900702106318957
-f1_scores: 0.6802420656474512
-time_taken_from_start: 21.604072332382202
-auc_score: 0.7457100293916334
+avg_loss: 0.5730699896812439
+total_acc: 69.52861952861953
+precisions: 0.7336375047795977
+recalls: 0.6952861952861953
+f1_scores: 0.6858177547541179
+time_taken_from_start: 28.49159860610962
+auc_score: 0.7738852057033876

roc_data.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1eec496eaf0223a6c65d0367eb586c968edc655b7e4c601d35db358f8419047d
-size 9437
+oid sha256:4c4af99c21a2122f6f4c4773439bbb77976243559acf78cd9b771f24d3ae9bdc
+size 5930

roc_data2.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:41fa9d96833c12979f8495141ee61c0ba07d4a20c5fb5bc18a7f72bf4d15e8fd
-size 28023

train_info.txt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ef4862f5c282efdfa49e13ed0f6cb344abcb7ae07fdfba535d48193bb8a3c1ed
-size 81939614