Repository: DOE-NCI-MOSSAIC/FrESCO (language: python)

Issue:
Update data_utils.py



Discussion / hints:

Fix patch:
diff --git a/configs/clc_step2.yml b/configs/clc_step2.yml
index 8712241..729979f 100644
--- a/configs/clc_step2.yml
+++ b/configs/clc_step2.yml
@@ -2,11 +2,11 @@ save_name: 'clc_test'  # foldX.h5 will be appended at the end; leave blank to au
 
 # dataloader settings
 data_kwargs:
-  model_path: '../savedmodels/clc_model/model.h5'
+  model_path: './savedmodels/clc_model/clc_model.h5'
   tasks: ["task_1", "task_2", "task_3", "task_4"]
   exclude_single: False
   shuffle_case_order: False
-  subset_proportion: 0.5
+  subset_proportion: 1.0
   reproducible: True
   random_seed: 42
 
@@ -42,4 +42,4 @@ abstain_kwargs:
   ntask_alpha: 0.1
   ntask_alpha_scale: 0.8
   ntask_max_abs: 0.9
-  ntask_min_acc: 0.975
\ No newline at end of file
+  ntask_min_acc: 0.975
diff --git a/fresco/validate/validate_params.py b/fresco/validate/validate_params.py
index 4d53631..9875692 100644
--- a/fresco/validate/validate_params.py
+++ b/fresco/validate/validate_params.py
@@ -35,7 +35,7 @@ def __init__(self, cli_args,
                 mod_args_file = cli_args.model_args
             else:
                 mod_args_file = 'model_args.yml'
-            print(cli_args.model_args)
+
             if os.path.isfile(mod_args_file):
                 with open(mod_args_file, "r", encoding="utf-8") as f_in:
                     self.model_args = yaml.safe_load(f_in)
@@ -299,7 +299,7 @@ class ValidateClcParams():
     def __init__(self, cli_args, data_source: str = 'pre-generated'):
 
         if len(cli_args.model_args) > 0:
-            mod_args_file = cli_args.args
+            mod_args_file = cli_args.model_args
         else:
             mod_args_file = 'clc_args.yml'
 
diff --git a/notebooks/clc_example.ipynb b/notebooks/clc_example.ipynb
new file mode 100644
index 0000000..b0c8c9e
--- /dev/null
+++ b/notebooks/clc_example.ipynb
@@ -0,0 +1,2138 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "7424773b",
+   "metadata": {},
+   "source": [
+    "This is an example for running the FrESCO library with a hierarchical version of the P3B3 becnhmark data from the ECP-Candle [repository](https://github.com/ECP-CANDLE/Benchmarks) with an added key `groups` which deginates how different entries are grouped together. Included within the FrESCO repository is a preformatted version of the clc dataset for model training. If you've not already done so, go to the data directory and unzip the dataset using the command `$ tar -xf clc.tar.gz`."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "460646d6",
+   "metadata": {},
+   "source": [
+    "Training a case-level ontext (clc) model is a two step process. Initially we'll need to train a model on the data. We can do this in a similar fashion to the other datasets. In the `configs/` directory are sample `model_args.yml` files for the three sample datasets, using the default settings in these files, we are ready to train a model for the first step."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "73c2a2b4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import fresco\n",
+    "import argparse"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ffa26c90",
+   "metadata": {},
+   "source": [
+    "The FrESCO library is typically run from the command line with arguments specifying the model type and model args, so we'll have to set them up manually for this notebook."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "5913468b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n",
+    "    _ = parser.add_argument(\"--model\", \"-m\", type=str, default='ie',\n",
+    "                        help=\"\"\"which type of model to create. Must be either\n",
+    "                                IE (information extraction) or clc (case-level context).\"\"\")\n",
+    "    _ = parser.add_argument('--model_path', '-mp', type=str, default='',\n",
+    "                       help=\"\"\"this is the location of the model\n",
+    "                               that will used to make predictions\"\"\")\n",
+    "    _ = parser.add_argument('--data_path', '-dp', type=str, default='',\n",
+    "                        help=\"\"\"where the data will load from. The default is\n",
+    "                                the path saved in the model\"\"\")\n",
+    "    _ = parser.add_argument('--model_args', '-args', type=str, default='',\n",
+    "                        help=\"\"\"file specifying the model or clc args; default is in\n",
+    "                                the fresco directory\"\"\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "24e7fe4b",
+   "metadata": {},
+   "source": [
+    "We are going to train a multi-task classification model on the clc dataset, the first step of which is an `information extraction` model. We'll also point the code to the clc model args file. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "85e7eddf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "args = parser.parse_args(args=['-m', 'ie', '-args', '../configs/clc_step1.yml'])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f1504aa2",
+   "metadata": {},
+   "source": [
+    "With these arguments specified, just need a few imports before we're ready to train our model. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "91cc5acb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from fresco import run_ie, run_clc\n",
+    "\n",
+    "from fresco.validate import exceptions"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "8a9fb57e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Validating kwargs in model_args.yml file\n",
+      "../configs/clc_step1.yml\n",
+      "Word embeddings file does not exist; will default to random embeddings.\n",
+      "Loading data and creating DataLoaders\n",
+      "Loading data from ../data/clc/\n",
+      "Num workers: 4, reproducible: True\n",
+      "Training on 15000 validate on 2000\n",
+      "\n",
+      "Defining a model\n",
+      "Creating model trainer\n",
+      "Training a mthisan model with 2 cuda device\n",
+      "\n",
+      "\n",
+      "epoch: 1\n",
+      "\n",
+      "training time 18.86\n",
+      "Training loss: 0.855519\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5233,     0.0615\n",
+      "      task_2:     0.5517,     0.4335\n",
+      "      task_3:     0.8872,     0.4742\n",
+      "      task_4:     0.4484,     0.3094\n",
+      "\n",
+      "epoch 1 validation\n",
+      "\n",
+      "epoch 1 val loss: 1.08947620, best val loss: inf\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1018\n",
+      "      task_2:     0.5830,     0.3895\n",
+      "      task_3:     0.8880,     0.4703\n",
+      "      task_4:     0.4780,     0.3428\n",
+      "\n",
+      "epoch: 2\n",
+      "\n",
+      "training time 17.45\n",
+      "Training loss: 0.847021\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5463,     0.1183\n",
+      "      task_2:     0.5479,     0.4688\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4473,     0.3032\n",
+      "\n",
+      "epoch 2 validation\n",
+      "\n",
+      "epoch 2 val loss: 0.97047849, best val loss: 1.08947620\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1187\n",
+      "      task_2:     0.5780,     0.3728\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4370,     0.2538\n",
+      "\n",
+      "epoch: 3\n",
+      "\n",
+      "training time 17.76\n",
+      "Training loss: 0.840675\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5470,     0.1180\n",
+      "      task_2:     0.5415,     0.4475\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4559,     0.3104\n",
+      "\n",
+      "epoch 3 validation\n",
+      "\n",
+      "epoch 3 val loss: 0.90129673, best val loss: 0.97047849\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1187\n",
+      "      task_2:     0.5810,     0.3825\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4140,     0.2124\n",
+      "\n",
+      "epoch: 4\n",
+      "\n",
+      "training time 17.91\n",
+      "Training loss: 0.719118\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5475,     0.1179\n",
+      "      task_2:     0.5493,     0.4529\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4489,     0.3014\n",
+      "\n",
+      "epoch 4 validation\n",
+      "\n",
+      "epoch 4 val loss: 0.85745309, best val loss: 0.90129673\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5440,     0.1188\n",
+      "      task_2:     0.5790,     0.3795\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4110,     0.2019\n",
+      "\n",
+      "epoch: 5\n",
+      "\n",
+      "training time 18.03\n",
+      "Training loss: 0.789439\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5481,     0.1192\n",
+      "      task_2:     0.5755,     0.4822\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4673,     0.3152\n",
+      "\n",
+      "epoch 5 validation\n",
+      "\n",
+      "epoch 5 val loss: 0.78731991, best val loss: 0.85745309\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.6100,     0.1974\n",
+      "      task_2:     0.7630,     0.7436\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.5280,     0.3742\n",
+      "\n",
+      "epoch: 6\n",
+      "\n",
+      "training time 18.19\n",
+      "Training loss: 0.710568\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.6147,     0.2031\n",
+      "      task_2:     0.8132,     0.8097\n",
+      "      task_3:     0.8934,     0.4737\n",
+      "      task_4:     0.4923,     0.3526\n",
+      "\n",
+      "epoch 6 validation\n",
+      "\n",
+      "epoch 6 val loss: 0.64467859, best val loss: 0.78731991\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.7240,     0.2634\n",
+      "      task_2:     0.8840,     0.8834\n",
+      "      task_3:     0.8880,     0.4703\n",
+      "      task_4:     0.5490,     0.4797\n",
+      "\n",
+      "epoch: 7\n",
+      "\n",
+      "training time 18.20\n",
+      "Training loss: 0.488941\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.7054,     0.2788\n",
+      "      task_2:     0.8868,     0.8843\n",
+      "      task_3:     0.8925,     0.4813\n",
+      "      task_4:     0.5458,     0.4739\n",
+      "\n",
+      "epoch 7 validation\n",
+      "\n",
+      "epoch 7 val loss: 0.55543782, best val loss: 0.64467859\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.7430,     0.2763\n",
+      "      task_2:     0.9230,     0.9217\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.5620,     0.5122\n",
+      "\n",
+      "epoch: 8\n",
+      "\n",
+      "training time 18.18\n",
+      "Training loss: 0.482716\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.7331,     0.3474\n",
+      "      task_2:     0.9027,     0.9000\n",
+      "      task_3:     0.8939,     0.4941\n",
+      "      task_4:     0.5576,     0.4874\n",
+      "\n",
+      "epoch 8 validation\n",
+      "\n",
+      "epoch 8 val loss: 0.51720239, best val loss: 0.55543782\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.7550,     0.3000\n",
+      "      task_2:     0.9250,     0.9232\n",
+      "      task_3:     0.8920,     0.5058\n",
+      "      task_4:     0.5600,     0.5245\n",
+      "\n",
+      "epoch: 9\n",
+      "\n",
+      "training time 18.21\n",
+      "Training loss: 0.468982\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.7574,     0.4211\n",
+      "      task_2:     0.9048,     0.9018\n",
+      "      task_3:     0.9003,     0.5648\n",
+      "      task_4:     0.5803,     0.5234\n",
+      "\n",
+      "epoch 9 validation\n",
+      "\n",
+      "epoch 9 val loss: 0.48643056, best val loss: 0.51720239\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.7750,     0.3857\n",
+      "      task_2:     0.9290,     0.9271\n",
+      "      task_3:     0.9030,     0.6394\n",
+      "      task_4:     0.5590,     0.5346\n",
+      "\n",
+      "epoch: 10\n",
+      "\n",
+      "training time 18.23\n",
+      "Training loss: 0.338182\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.7787,     0.4764\n",
+      "      task_2:     0.9060,     0.9031\n",
+      "      task_3:     0.9146,     0.6660\n",
+      "      task_4:     0.6221,     0.5684\n",
+      "\n",
+      "epoch 10 validation\n",
+      "\n",
+      "epoch 10 val loss: 0.44372915, best val loss: 0.48643056\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.7960,     0.4683\n",
+      "      task_2:     0.9310,     0.9291\n",
+      "      task_3:     0.9200,     0.8064\n",
+      "      task_4:     0.6260,     0.6111\n",
+      "\n",
+      "epoch: 11\n",
+      "\n",
+      "training time 18.27\n",
+      "Training loss: 0.359736\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.8017,     0.5560\n",
+      "      task_2:     0.9127,     0.9100\n",
+      "      task_3:     0.9377,     0.7921\n",
+      "      task_4:     0.6621,     0.6205\n",
+      "\n",
+      "epoch 11 validation\n",
+      "\n",
+      "epoch 11 val loss: 0.40871409, best val loss: 0.44372915\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.8050,     0.5326\n",
+      "      task_2:     0.9270,     0.9246\n",
+      "      task_3:     0.9220,     0.8298\n",
+      "      task_4:     0.6680,     0.6511\n",
+      "\n",
+      "epoch: 12\n",
+      "\n",
+      "training time 18.25\n",
+      "Training loss: 0.422298\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.8295,     0.6444\n",
+      "      task_2:     0.9139,     0.9113\n",
+      "      task_3:     0.9508,     0.8486\n",
+      "      task_4:     0.6945,     0.6577\n",
+      "\n",
+      "epoch 12 validation\n",
+      "\n",
+      "epoch 12 val loss: 0.37391173, best val loss: 0.40871409\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.8350,     0.6499\n",
+      "      task_2:     0.9340,     0.9321\n",
+      "      task_3:     0.9270,     0.8470\n",
+      "      task_4:     0.7110,     0.6914\n",
+      "\n",
+      "epoch: 13\n",
+      "\n",
+      "training time 18.29\n",
+      "Training loss: 0.213347\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.8488,     0.6983\n",
+      "      task_2:     0.9149,     0.9125\n",
+      "      task_3:     0.9628,     0.8909\n",
+      "      task_4:     0.7155,     0.6810\n",
+      "\n",
+      "epoch 13 validation\n",
+      "\n",
+      "epoch 13 val loss: 0.36525388, best val loss: 0.37391173\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.8600,     0.7089\n",
+      "      task_2:     0.9340,     0.9323\n",
+      "      task_3:     0.9330,     0.8588\n",
+      "      task_4:     0.7330,     0.7152\n",
+      "\n",
+      "epoch: 14\n",
+      "\n",
+      "training time 18.29\n",
+      "Training loss: 0.288499\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.8613,     0.7272\n",
+      "      task_2:     0.9179,     0.9156\n",
+      "      task_3:     0.9661,     0.9030\n",
+      "      task_4:     0.7416,     0.7127\n",
+      "\n",
+      "epoch 14 validation\n",
+      "\n",
+      "epoch 14 val loss: 0.34798697, best val loss: 0.36525388\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.8760,     0.7575\n",
+      "      task_2:     0.9360,     0.9344\n",
+      "      task_3:     0.9400,     0.8723\n",
+      "      task_4:     0.7530,     0.7310\n",
+      "\n",
+      "epoch: 15\n",
+      "\n",
+      "training time 18.29\n",
+      "Training loss: 0.286955\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.8739,     0.7627\n",
+      "      task_2:     0.9195,     0.9172\n",
+      "      task_3:     0.9699,     0.9146\n",
+      "      task_4:     0.7600,     0.7297\n",
+      "\n",
+      "epoch 15 validation\n",
+      "\n",
+      "epoch 15 val loss: 0.33579794, best val loss: 0.34798697\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.8840,     0.7733\n",
+      "      task_2:     0.9350,     0.9331\n",
+      "      task_3:     0.9440,     0.8801\n",
+      "      task_4:     0.7870,     0.7677\n",
+      "\n",
+      "epoch: 16\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "training time 18.27\n",
+      "Training loss: 0.211272\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.8859,     0.7868\n",
+      "      task_2:     0.9173,     0.9151\n",
+      "      task_3:     0.9717,     0.9209\n",
+      "      task_4:     0.7873,     0.7607\n",
+      "\n",
+      "epoch 16 validation\n",
+      "\n",
+      "epoch 16 val loss: 0.32126487, best val loss: 0.33579794\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.8980,     0.8099\n",
+      "      task_2:     0.9330,     0.9309\n",
+      "      task_3:     0.9480,     0.8873\n",
+      "      task_4:     0.7990,     0.7785\n",
+      "\n",
+      "epoch: 17\n",
+      "\n",
+      "training time 18.25\n",
+      "Training loss: 0.208659\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.8979,     0.8123\n",
+      "      task_2:     0.9223,     0.9203\n",
+      "      task_3:     0.9735,     0.9258\n",
+      "      task_4:     0.8068,     0.7828\n",
+      "\n",
+      "epoch 17 validation\n",
+      "\n",
+      "epoch 17 val loss: 0.31447380, best val loss: 0.32126487\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9040,     0.8170\n",
+      "      task_2:     0.9300,     0.9277\n",
+      "      task_3:     0.9530,     0.8964\n",
+      "      task_4:     0.8080,     0.7888\n",
+      "\n",
+      "epoch: 18\n",
+      "\n",
+      "training time 18.24\n",
+      "Training loss: 0.324498\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9035,     0.8220\n",
+      "      task_2:     0.9213,     0.9191\n",
+      "      task_3:     0.9761,     0.9341\n",
+      "      task_4:     0.8239,     0.8011\n",
+      "\n",
+      "epoch 18 validation\n",
+      "\n",
+      "epoch 18 val loss: 0.30711931, best val loss: 0.31447380\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9140,     0.8388\n",
+      "      task_2:     0.9300,     0.9282\n",
+      "      task_3:     0.9560,     0.9034\n",
+      "      task_4:     0.8230,     0.8026\n",
+      "\n",
+      "epoch: 19\n",
+      "\n",
+      "training time 18.25\n",
+      "Training loss: 0.251227\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9086,     0.8316\n",
+      "      task_2:     0.9217,     0.9196\n",
+      "      task_3:     0.9785,     0.9404\n",
+      "      task_4:     0.8370,     0.8206\n",
+      "\n",
+      "epoch 19 validation\n",
+      "\n",
+      "epoch 19 val loss: 0.29742832, best val loss: 0.30711931\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9220,     0.8536\n",
+      "      task_2:     0.9320,     0.9298\n",
+      "      task_3:     0.9600,     0.9110\n",
+      "      task_4:     0.8430,     0.8224\n",
+      "\n",
+      "epoch: 20\n",
+      "\n",
+      "training time 18.25\n",
+      "Training loss: 0.190083\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9201,     0.8538\n",
+      "      task_2:     0.9231,     0.9211\n",
+      "      task_3:     0.9793,     0.9430\n",
+      "      task_4:     0.8472,     0.8306\n",
+      "\n",
+      "epoch 20 validation\n",
+      "\n",
+      "epoch 20 val loss: 0.29079486, best val loss: 0.29742832\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9260,     0.8646\n",
+      "      task_2:     0.9320,     0.9300\n",
+      "      task_3:     0.9620,     0.9149\n",
+      "      task_4:     0.8540,     0.8331\n",
+      "\n",
+      "epoch: 21\n",
+      "\n",
+      "training time 18.27\n",
+      "Training loss: 0.209531\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9210,     0.8613\n",
+      "      task_2:     0.9229,     0.9210\n",
+      "      task_3:     0.9795,     0.9437\n",
+      "      task_4:     0.8611,     0.8453\n",
+      "\n",
+      "epoch 21 validation\n",
+      "\n",
+      "epoch 21 val loss: 0.29132099, best val loss: 0.29079486\n",
+      "patience counter is at 1 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9300,     0.8725\n",
+      "      task_2:     0.9350,     0.9337\n",
+      "      task_3:     0.9610,     0.9129\n",
+      "      task_4:     0.8660,     0.8469\n",
+      "\n",
+      "epoch: 22\n",
+      "\n",
+      "training time 18.28\n",
+      "Training loss: 0.160553\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9259,     0.8733\n",
+      "      task_2:     0.9275,     0.9257\n",
+      "      task_3:     0.9799,     0.9452\n",
+      "      task_4:     0.8700,     0.8543\n",
+      "\n",
+      "epoch 22 validation\n",
+      "\n",
+      "epoch 22 val loss: 0.28995841, best val loss: 0.29079486\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9290,     0.8736\n",
+      "      task_2:     0.9320,     0.9302\n",
+      "      task_3:     0.9580,     0.9072\n",
+      "      task_4:     0.8740,     0.8585\n",
+      "\n",
+      "epoch: 23\n",
+      "\n",
+      "training time 18.27\n",
+      "Training loss: 0.136115\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9275,     0.8743\n",
+      "      task_2:     0.9263,     0.9245\n",
+      "      task_3:     0.9819,     0.9507\n",
+      "      task_4:     0.8749,     0.8607\n",
+      "\n",
+      "epoch 23 validation\n",
+      "\n",
+      "epoch 23 val loss: 0.29575746, best val loss: 0.28995841\n",
+      "patience counter is at 1 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9340,     0.8828\n",
+      "      task_2:     0.9340,     0.9323\n",
+      "      task_3:     0.9620,     0.9149\n",
+      "      task_4:     0.8680,     0.8525\n",
+      "\n",
+      "epoch: 24\n",
+      "\n",
+      "training time 18.29\n",
+      "Training loss: 0.124581\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9314,     0.8834\n",
+      "      task_2:     0.9283,     0.9266\n",
+      "      task_3:     0.9836,     0.9555\n",
+      "      task_4:     0.8821,     0.8676\n",
+      "\n",
+      "epoch 24 validation\n",
+      "\n",
+      "epoch 24 val loss: 0.28823978, best val loss: 0.28995841\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9360,     0.8864\n",
+      "      task_2:     0.9370,     0.9354\n",
+      "      task_3:     0.9650,     0.9208\n",
+      "      task_4:     0.8850,     0.8689\n",
+      "\n",
+      "epoch: 25\n",
+      "\n",
+      "training time 18.29\n",
+      "Training loss: 0.168261\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9317,     0.8828\n",
+      "      task_2:     0.9284,     0.9267\n",
+      "      task_3:     0.9837,     0.9555\n",
+      "      task_4:     0.8902,     0.8758\n",
+      "\n",
+      "epoch 25 validation\n",
+      "\n",
+      "epoch 25 val loss: 0.27900909, best val loss: 0.28823978\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9360,     0.8864\n",
+      "      task_2:     0.9370,     0.9354\n",
+      "      task_3:     0.9650,     0.9208\n",
+      "      task_4:     0.8940,     0.8816\n",
+      "\n",
+      "epoch: 26\n",
+      "\n",
+      "training time 18.29\n",
+      "Training loss: 0.179235\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9342,     0.8885\n",
+      "      task_2:     0.9305,     0.9289\n",
+      "      task_3:     0.9831,     0.9544\n",
+      "      task_4:     0.8940,     0.8817\n",
+      "\n",
+      "epoch 26 validation\n",
+      "\n",
+      "epoch 26 val loss: 0.28270503, best val loss: 0.27900909\n",
+      "patience counter is at 1 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9370,     0.8854\n",
+      "      task_2:     0.9400,     0.9385\n",
+      "      task_3:     0.9680,     0.9269\n",
+      "      task_4:     0.8980,     0.8856\n",
+      "\n",
+      "epoch: 27\n",
+      "\n",
+      "training time 18.25\n",
+      "Training loss: 0.030038\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9355,     0.8897\n",
+      "      task_2:     0.9299,     0.9282\n",
+      "      task_3:     0.9833,     0.9548\n",
+      "      task_4:     0.9006,     0.8891\n",
+      "\n",
+      "epoch 27 validation\n",
+      "\n",
+      "epoch 27 val loss: 0.28798433, best val loss: 0.27900909\n",
+      "patience counter is at 2 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9390,     0.8877\n",
+      "      task_2:     0.9360,     0.9343\n",
+      "      task_3:     0.9630,     0.9168\n",
+      "      task_4:     0.8970,     0.8834\n",
+      "\n",
+      "epoch: 28\n",
+      "\n",
+      "training time 18.32\n",
+      "Training loss: 0.096912\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9399,     0.8968\n",
+      "      task_2:     0.9333,     0.9317\n",
+      "      task_3:     0.9831,     0.9541\n",
+      "      task_4:     0.9057,     0.8937\n",
+      "\n",
+      "epoch 28 validation\n",
+      "\n",
+      "epoch 28 val loss: 0.27779495, best val loss: 0.27900909\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9370,     0.8874\n",
+      "      task_2:     0.9410,     0.9396\n",
+      "      task_3:     0.9680,     0.9274\n",
+      "      task_4:     0.9070,     0.8950\n",
+      "\n",
+      "epoch: 29\n",
+      "\n",
+      "training time 18.29\n",
+      "Training loss: 0.093401\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9413,     0.9010\n",
+      "      task_2:     0.9322,     0.9306\n",
+      "      task_3:     0.9837,     0.9560\n",
+      "      task_4:     0.9069,     0.8952\n",
+      "\n",
+      "epoch 29 validation\n",
+      "\n",
+      "epoch 29 val loss: 0.27517348, best val loss: 0.27779495\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9410,     0.8901\n",
+      "      task_2:     0.9450,     0.9438\n",
+      "      task_3:     0.9700,     0.9314\n",
+      "      task_4:     0.9080,     0.8970\n",
+      "\n",
+      "epoch: 30\n",
+      "\n",
+      "training time 18.26\n",
+      "Training loss: 0.146989\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9461,     0.9108\n",
+      "      task_2:     0.9333,     0.9318\n",
+      "      task_3:     0.9847,     0.9587\n",
+      "      task_4:     0.9130,     0.9025\n",
+      "\n",
+      "epoch 30 validation\n",
+      "\n",
+      "epoch 30 val loss: 0.26750629, best val loss: 0.27517348\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9420,     0.8951\n",
+      "      task_2:     0.9460,     0.9447\n",
+      "      task_3:     0.9690,     0.9289\n",
+      "      task_4:     0.9150,     0.9027\n",
+      "\n",
+      "epoch: 31\n",
+      "\n",
+      "training time 18.28\n",
+      "Training loss: 0.081953\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9469,     0.9119\n",
+      "      task_2:     0.9310,     0.9295\n",
+      "      task_3:     0.9865,     0.9635\n",
+      "      task_4:     0.9173,     0.9095\n",
+      "\n",
+      "epoch 31 validation\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "epoch 31 val loss: 0.27292886, best val loss: 0.26750629\n",
+      "patience counter is at 1 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9410,     0.8891\n",
+      "      task_2:     0.9450,     0.9436\n",
+      "      task_3:     0.9680,     0.9274\n",
+      "      task_4:     0.9170,     0.9065\n",
+      "\n",
+      "epoch: 32\n",
+      "\n",
+      "training time 18.29\n",
+      "Training loss: 0.239570\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9487,     0.9151\n",
+      "      task_2:     0.9346,     0.9331\n",
+      "      task_3:     0.9861,     0.9623\n",
+      "      task_4:     0.9185,     0.9099\n",
+      "\n",
+      "epoch 32 validation\n",
+      "\n",
+      "epoch 32 val loss: 0.27532295, best val loss: 0.26750629\n",
+      "patience counter is at 2 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9450,     0.8968\n",
+      "      task_2:     0.9470,     0.9457\n",
+      "      task_3:     0.9680,     0.9274\n",
+      "      task_4:     0.9110,     0.8988\n",
+      "\n",
+      "epoch: 33\n",
+      "\n",
+      "training time 18.26\n",
+      "Training loss: 0.101796\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9469,     0.9121\n",
+      "      task_2:     0.9383,     0.9369\n",
+      "      task_3:     0.9861,     0.9627\n",
+      "      task_4:     0.9225,     0.9136\n",
+      "\n",
+      "epoch 33 validation\n",
+      "\n",
+      "epoch 33 val loss: 0.26104848, best val loss: 0.26750629\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9460,     0.9024\n",
+      "      task_2:     0.9460,     0.9446\n",
+      "      task_3:     0.9720,     0.9351\n",
+      "      task_4:     0.9160,     0.9049\n",
+      "\n",
+      "epoch: 34\n",
+      "\n",
+      "training time 18.26\n",
+      "Training loss: 0.246199\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9525,     0.9227\n",
+      "      task_2:     0.9403,     0.9389\n",
+      "      task_3:     0.9875,     0.9664\n",
+      "      task_4:     0.9231,     0.9144\n",
+      "\n",
+      "epoch 34 validation\n",
+      "\n",
+      "epoch 34 val loss: 0.26378720, best val loss: 0.26104848\n",
+      "patience counter is at 1 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9430,     0.8956\n",
+      "      task_2:     0.9470,     0.9460\n",
+      "      task_3:     0.9720,     0.9351\n",
+      "      task_4:     0.9190,     0.9080\n",
+      "\n",
+      "epoch: 35\n",
+      "\n",
+      "training time 18.27\n",
+      "Training loss: 0.055518\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9547,     0.9264\n",
+      "      task_2:     0.9389,     0.9376\n",
+      "      task_3:     0.9873,     0.9660\n",
+      "      task_4:     0.9267,     0.9189\n",
+      "\n",
+      "epoch 35 validation\n",
+      "\n",
+      "epoch 35 val loss: 0.26383388, best val loss: 0.26104848\n",
+      "patience counter is at 2 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9480,     0.9042\n",
+      "      task_2:     0.9510,     0.9499\n",
+      "      task_3:     0.9700,     0.9310\n",
+      "      task_4:     0.9170,     0.9054\n",
+      "\n",
+      "epoch: 36\n",
+      "\n",
+      "training time 18.29\n",
+      "Training loss: 0.099308\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9570,     0.9303\n",
+      "      task_2:     0.9391,     0.9378\n",
+      "      task_3:     0.9879,     0.9677\n",
+      "      task_4:     0.9297,     0.9233\n",
+      "\n",
+      "epoch 36 validation\n",
+      "\n",
+      "epoch 36 val loss: 0.26765602, best val loss: 0.26104848\n",
+      "patience counter is at 3 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9510,     0.9117\n",
+      "      task_2:     0.9510,     0.9498\n",
+      "      task_3:     0.9720,     0.9351\n",
+      "      task_4:     0.9150,     0.9031\n",
+      "\n",
+      "epoch: 37\n",
+      "\n",
+      "training time 18.27\n",
+      "Training loss: 0.075664\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9550,     0.9294\n",
+      "      task_2:     0.9411,     0.9398\n",
+      "      task_3:     0.9873,     0.9658\n",
+      "      task_4:     0.9321,     0.9253\n",
+      "\n",
+      "epoch 37 validation\n",
+      "\n",
+      "epoch 37 val loss: 0.26899231, best val loss: 0.26104848\n",
+      "patience counter is at 4 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9470,     0.9036\n",
+      "      task_2:     0.9500,     0.9488\n",
+      "      task_3:     0.9690,     0.9294\n",
+      "      task_4:     0.9180,     0.9073\n",
+      "\n",
+      "epoch: 38\n",
+      "\n",
+      "training time 18.28\n",
+      "Training loss: 0.224249\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9553,     0.9289\n",
+      "      task_2:     0.9410,     0.9397\n",
+      "      task_3:     0.9881,     0.9682\n",
+      "      task_4:     0.9342,     0.9273\n",
+      "\n",
+      "epoch 38 validation\n",
+      "\n",
+      "epoch 38 val loss: 0.25800124, best val loss: 0.26104848\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9500,     0.9102\n",
+      "      task_2:     0.9490,     0.9478\n",
+      "      task_3:     0.9720,     0.9351\n",
+      "      task_4:     0.9220,     0.9115\n",
+      "\n",
+      "epoch: 39\n",
+      "\n",
+      "training time 18.35\n",
+      "Training loss: 0.091395\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9575,     0.9322\n",
+      "      task_2:     0.9439,     0.9427\n",
+      "      task_3:     0.9883,     0.9687\n",
+      "      task_4:     0.9354,     0.9290\n",
+      "\n",
+      "epoch 39 validation\n",
+      "\n",
+      "epoch 39 val loss: 0.25166786, best val loss: 0.25800124\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9500,     0.9102\n",
+      "      task_2:     0.9510,     0.9500\n",
+      "      task_3:     0.9730,     0.9372\n",
+      "      task_4:     0.9250,     0.9147\n",
+      "\n",
+      "epoch: 40\n",
+      "\n",
+      "training time 18.28\n",
+      "Training loss: 0.116261\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9577,     0.9321\n",
+      "      task_2:     0.9412,     0.9399\n",
+      "      task_3:     0.9887,     0.9698\n",
+      "      task_4:     0.9369,     0.9325\n",
+      "\n",
+      "epoch 40 validation\n",
+      "\n",
+      "epoch 40 val loss: 0.25270403, best val loss: 0.25166786\n",
+      "patience counter is at 1 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9530,     0.9135\n",
+      "      task_2:     0.9490,     0.9477\n",
+      "      task_3:     0.9710,     0.9330\n",
+      "      task_4:     0.9260,     0.9147\n",
+      "\n",
+      "epoch: 41\n",
+      "\n",
+      "training time 18.25\n",
+      "Training loss: 0.123772\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9592,     0.9353\n",
+      "      task_2:     0.9465,     0.9453\n",
+      "      task_3:     0.9880,     0.9679\n",
+      "      task_4:     0.9402,     0.9337\n",
+      "\n",
+      "epoch 41 validation\n",
+      "\n",
+      "epoch 41 val loss: 0.24653343, best val loss: 0.25166786\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9520,     0.9130\n",
+      "      task_2:     0.9530,     0.9521\n",
+      "      task_3:     0.9750,     0.9414\n",
+      "      task_4:     0.9250,     0.9155\n",
+      "\n",
+      "epoch: 42\n",
+      "\n",
+      "training time 18.27\n",
+      "Training loss: 0.126785\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9595,     0.9371\n",
+      "      task_2:     0.9434,     0.9422\n",
+      "      task_3:     0.9887,     0.9696\n",
+      "      task_4:     0.9395,     0.9348\n",
+      "\n",
+      "epoch 42 validation\n",
+      "\n",
+      "epoch 42 val loss: 0.24145810, best val loss: 0.24653343\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9540,     0.9169\n",
+      "      task_2:     0.9510,     0.9498\n",
+      "      task_3:     0.9750,     0.9414\n",
+      "      task_4:     0.9270,     0.9165\n",
+      "\n",
+      "epoch: 43\n",
+      "\n",
+      "training time 18.32\n",
+      "Training loss: 0.133249\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9624,     0.9420\n",
+      "      task_2:     0.9489,     0.9478\n",
+      "      task_3:     0.9875,     0.9667\n",
+      "      task_4:     0.9387,     0.9339\n",
+      "\n",
+      "epoch 43 validation\n",
+      "\n",
+      "epoch 43 val loss: 0.23601118, best val loss: 0.24145810\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9520,     0.9133\n",
+      "      task_2:     0.9550,     0.9541\n",
+      "      task_3:     0.9790,     0.9501\n",
+      "      task_4:     0.9270,     0.9174\n",
+      "\n",
+      "epoch: 44\n",
+      "\n",
+      "training time 18.26\n",
+      "Training loss: 0.155977\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9637,     0.9404\n",
+      "      task_2:     0.9503,     0.9492\n",
+      "      task_3:     0.9891,     0.9709\n",
+      "      task_4:     0.9439,     0.9379\n",
+      "\n",
+      "epoch 44 validation\n",
+      "\n",
+      "epoch 44 val loss: 0.24147267, best val loss: 0.23601118\n",
+      "patience counter is at 1 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9540,     0.9158\n",
+      "      task_2:     0.9520,     0.9508\n",
+      "      task_3:     0.9740,     0.9393\n",
+      "      task_4:     0.9320,     0.9219\n",
+      "\n",
+      "epoch: 45\n",
+      "\n",
+      "training time 18.28\n",
+      "Training loss: 0.072216\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9624,     0.9415\n",
+      "      task_2:     0.9470,     0.9459\n",
+      "      task_3:     0.9893,     0.9713\n",
+      "      task_4:     0.9415,     0.9366\n",
+      "\n",
+      "epoch 45 validation\n",
+      "\n",
+      "epoch 45 val loss: 0.23799448, best val loss: 0.23601118\n",
+      "patience counter is at 2 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9530,     0.9152\n",
+      "      task_2:     0.9560,     0.9550\n",
+      "      task_3:     0.9750,     0.9414\n",
+      "      task_4:     0.9300,     0.9191\n",
+      "\n",
+      "epoch: 46\n",
+      "\n",
+      "training time 18.29\n",
+      "Training loss: 0.148015\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9639,     0.9429\n",
+      "      task_2:     0.9483,     0.9472\n",
+      "      task_3:     0.9893,     0.9714\n",
+      "      task_4:     0.9448,     0.9389\n",
+      "\n",
+      "epoch 46 validation\n",
+      "\n",
+      "epoch 46 val loss: 0.24089598, best val loss: 0.23601118\n",
+      "patience counter is at 3 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9530,     0.9176\n",
+      "      task_2:     0.9550,     0.9540\n",
+      "      task_3:     0.9790,     0.9501\n",
+      "      task_4:     0.9300,     0.9199\n",
+      "\n",
+      "epoch: 47\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "training time 18.28\n",
+      "Training loss: 0.204627\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9658,     0.9471\n",
+      "      task_2:     0.9477,     0.9466\n",
+      "      task_3:     0.9899,     0.9728\n",
+      "      task_4:     0.9447,     0.9393\n",
+      "\n",
+      "epoch 47 validation\n",
+      "\n",
+      "epoch 47 val loss: 0.23976292, best val loss: 0.23601118\n",
+      "patience counter is at 4 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9560,     0.9220\n",
+      "      task_2:     0.9540,     0.9529\n",
+      "      task_3:     0.9740,     0.9393\n",
+      "      task_4:     0.9290,     0.9201\n",
+      "\n",
+      "epoch: 48\n",
+      "\n",
+      "training time 18.30\n",
+      "Training loss: 0.110330\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9651,     0.9456\n",
+      "      task_2:     0.9504,     0.9494\n",
+      "      task_3:     0.9892,     0.9711\n",
+      "      task_4:     0.9467,     0.9420\n",
+      "\n",
+      "epoch 48 validation\n",
+      "\n",
+      "epoch 48 val loss: 0.23263797, best val loss: 0.23601118\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9560,     0.9208\n",
+      "      task_2:     0.9540,     0.9529\n",
+      "      task_3:     0.9770,     0.9457\n",
+      "      task_4:     0.9320,     0.9213\n",
+      "\n",
+      "epoch: 49\n",
+      "\n",
+      "training time 18.23\n",
+      "Training loss: 0.039630\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9657,     0.9467\n",
+      "      task_2:     0.9521,     0.9511\n",
+      "      task_3:     0.9896,     0.9722\n",
+      "      task_4:     0.9452,     0.9416\n",
+      "\n",
+      "epoch 49 validation\n",
+      "\n",
+      "epoch 49 val loss: 0.23426842, best val loss: 0.23263797\n",
+      "patience counter is at 1 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9520,     0.9155\n",
+      "      task_2:     0.9550,     0.9540\n",
+      "      task_3:     0.9790,     0.9501\n",
+      "      task_4:     0.9340,     0.9259\n",
+      "\n",
+      "epoch: 50\n",
+      "\n",
+      "training time 18.32\n",
+      "Training loss: 0.126921\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9678,     0.9499\n",
+      "      task_2:     0.9544,     0.9535\n",
+      "      task_3:     0.9897,     0.9726\n",
+      "      task_4:     0.9461,     0.9420\n",
+      "\n",
+      "epoch 50 validation\n",
+      "\n",
+      "epoch 50 val loss: 0.23213682, best val loss: 0.23263797\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9590,     0.9244\n",
+      "      task_2:     0.9540,     0.9530\n",
+      "      task_3:     0.9760,     0.9436\n",
+      "      task_4:     0.9370,     0.9289\n",
+      "\n",
+      "epoch: 51\n",
+      "\n",
+      "training time 18.32\n",
+      "Training loss: 0.042747\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9677,     0.9496\n",
+      "      task_2:     0.9539,     0.9529\n",
+      "      task_3:     0.9906,     0.9749\n",
+      "      task_4:     0.9485,     0.9433\n",
+      "\n",
+      "epoch 51 validation\n",
+      "\n",
+      "epoch 51 val loss: 0.23898021, best val loss: 0.23213682\n",
+      "patience counter is at 1 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9590,     0.9259\n",
+      "      task_2:     0.9540,     0.9530\n",
+      "      task_3:     0.9800,     0.9523\n",
+      "      task_4:     0.9370,     0.9295\n",
+      "\n",
+      "epoch: 52\n",
+      "\n",
+      "training time 18.29\n",
+      "Training loss: 0.061133\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9666,     0.9473\n",
+      "      task_2:     0.9559,     0.9550\n",
+      "      task_3:     0.9912,     0.9765\n",
+      "      task_4:     0.9494,     0.9448\n",
+      "\n",
+      "epoch 52 validation\n",
+      "\n",
+      "epoch 52 val loss: 0.22615379, best val loss: 0.23213682\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9590,     0.9272\n",
+      "      task_2:     0.9530,     0.9520\n",
+      "      task_3:     0.9810,     0.9545\n",
+      "      task_4:     0.9380,     0.9297\n",
+      "\n",
+      "epoch: 53\n",
+      "\n",
+      "training time 18.28\n",
+      "Training loss: 0.111143\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9674,     0.9494\n",
+      "      task_2:     0.9535,     0.9525\n",
+      "      task_3:     0.9924,     0.9798\n",
+      "      task_4:     0.9511,     0.9465\n",
+      "\n",
+      "epoch 53 validation\n",
+      "\n",
+      "epoch 53 val loss: 0.21991716, best val loss: 0.22615379\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9570,     0.9258\n",
+      "      task_2:     0.9560,     0.9550\n",
+      "      task_3:     0.9790,     0.9501\n",
+      "      task_4:     0.9380,     0.9300\n",
+      "\n",
+      "epoch: 54\n",
+      "\n",
+      "training time 18.27\n",
+      "Training loss: 0.061330\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9679,     0.9509\n",
+      "      task_2:     0.9550,     0.9541\n",
+      "      task_3:     0.9907,     0.9751\n",
+      "      task_4:     0.9537,     0.9494\n",
+      "\n",
+      "epoch 54 validation\n",
+      "\n",
+      "epoch 54 val loss: 0.22515912, best val loss: 0.21991716\n",
+      "patience counter is at 1 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9590,     0.9265\n",
+      "      task_2:     0.9570,     0.9561\n",
+      "      task_3:     0.9740,     0.9393\n",
+      "      task_4:     0.9410,     0.9320\n",
+      "\n",
+      "epoch: 55\n",
+      "\n",
+      "training time 18.34\n",
+      "Training loss: 0.051121\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9695,     0.9537\n",
+      "      task_2:     0.9581,     0.9573\n",
+      "      task_3:     0.9909,     0.9757\n",
+      "      task_4:     0.9515,     0.9484\n",
+      "\n",
+      "epoch 55 validation\n",
+      "\n",
+      "epoch 55 val loss: 0.22365610, best val loss: 0.21991716\n",
+      "patience counter is at 2 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9540,     0.9184\n",
+      "      task_2:     0.9570,     0.9562\n",
+      "      task_3:     0.9740,     0.9393\n",
+      "      task_4:     0.9420,     0.9328\n",
+      "\n",
+      "epoch: 56\n",
+      "\n",
+      "training time 18.33\n",
+      "Training loss: 0.079727\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9713,     0.9570\n",
+      "      task_2:     0.9595,     0.9587\n",
+      "      task_3:     0.9913,     0.9768\n",
+      "      task_4:     0.9553,     0.9515\n",
+      "\n",
+      "epoch 56 validation\n",
+      "\n",
+      "epoch 56 val loss: 0.23164195, best val loss: 0.21991716\n",
+      "patience counter is at 3 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9580,     0.9257\n",
+      "      task_2:     0.9570,     0.9559\n",
+      "      task_3:     0.9730,     0.9372\n",
+      "      task_4:     0.9390,     0.9309\n",
+      "\n",
+      "epoch: 57\n",
+      "\n",
+      "training time 18.30\n",
+      "Training loss: 0.067694\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9698,     0.9543\n",
+      "      task_2:     0.9579,     0.9570\n",
+      "      task_3:     0.9901,     0.9738\n",
+      "      task_4:     0.9546,     0.9514\n",
+      "\n",
+      "epoch 57 validation\n",
+      "\n",
+      "epoch 57 val loss: 0.22140498, best val loss: 0.21991716\n",
+      "patience counter is at 4 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9600,     0.9294\n",
+      "      task_2:     0.9580,     0.9570\n",
+      "      task_3:     0.9830,     0.9590\n",
+      "      task_4:     0.9420,     0.9348\n",
+      "\n",
+      "epoch: 58\n",
+      "\n",
+      "training time 18.33\n",
+      "Training loss: 0.081083\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9707,     0.9557\n",
+      "      task_2:     0.9585,     0.9577\n",
+      "      task_3:     0.9912,     0.9765\n",
+      "      task_4:     0.9557,     0.9522\n",
+      "\n",
+      "epoch 58 validation\n",
+      "\n",
+      "epoch 58 val loss: 0.22539188, best val loss: 0.21991716\n",
+      "patience counter is at 5 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.9620,     0.9317\n",
+      "      task_2:     0.9560,     0.9551\n",
+      "      task_3:     0.9740,     0.9393\n",
+      "      task_4:     0.9370,     0.9297\n",
+      "saving to savedmodels/clc_model/clc_model_fold0.h5\n",
+      "\n",
+      "Scoring test set\n",
+      "\n",
+      "Predicting train set\n",
+      "\n",
+      "Predicting val set\n",
+      "\n",
+      "Predicting test set\n",
+      "Saving predictions to csv\n",
+      "\n",
+      "Evaluating train set\n",
+      "\n",
+      "Evaluating val set\n",
+      "\n",
+      "Evaluating test set\n",
+      "Saving predictions to csv\n",
+      "\n",
+      "Full model file has been saved at savedmodels/clc_model/clc_model_20230523212527_fold0.h5\n"
+     ]
+    }
+   ],
+   "source": [
+    "run_ie.run_ie(args)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e6e548e0",
+   "metadata": {},
+   "source": [
+    "Now that we've trained an inital model, we're ready to train the final clc model. First we'll rename the saved model from step one, then specify the updated name in the args for step two. The default name for step two is: `clc_model.h5`.  Now we're on our way."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "93164fe9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "args = parser.parse_args(args=['-m', 'clc', '-args', '../configs/clc_step2.yml'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "3ecefce5",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Validating kwargs in clc_args.yml file\n",
+      "Loading trained model from ./savedmodels/clc_model/clc_model.h5\n",
+      "Word embeddings file does not exist; will default to random embeddings.\n",
+      "Validating kwargs from pretrained model \n",
+      "Word embeddings file does not exist; will default to random embeddings.\n",
+      "Loading data and creating DataLoaders\n",
+      "Loading data from ../data/clc/\n",
+      "Num workers: 4, reproducible: True\n",
+      "Training on 15000 validate on 2000\n",
+      "model loaded\n",
+      "\n",
+      "Defining a CLC model\n",
+      "Creating model trainer\n",
+      "Training a case-level model with 2 cuda device\n",
+      "\n",
+      "\n",
+      "epoch: 1\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.931656\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.4310,     0.0540\n",
+      "      task_2:     0.5139,     0.4975\n",
+      "      task_3:     0.8729,     0.4909\n",
+      "      task_4:     0.4243,     0.3267\n",
+      "\n",
+      "epoch 1 validation\n",
+      "\n",
+      "epoch 1 val loss: 0.90204819, best val loss: inf\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5570,     0.4368\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4215,     0.3000\n",
+      "\n",
+      "epoch: 2\n",
+      "\n",
+      "training time 1.50\n",
+      "Training loss: 0.913104\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5228,     0.0532\n",
+      "      task_2:     0.5197,     0.4922\n",
+      "      task_3:     0.8920,     0.4727\n",
+      "      task_4:     0.4343,     0.3132\n",
+      "\n",
+      "epoch 2 validation\n",
+      "\n",
+      "epoch 2 val loss: 0.88555384, best val loss: 0.90204819\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5570,     0.4368\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4230,     0.2962\n",
+      "\n",
+      "epoch: 3\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.880551\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5295,     0.1132\n",
+      "      task_2:     0.5251,     0.4932\n",
+      "      task_3:     0.8917,     0.4744\n",
+      "      task_4:     0.4408,     0.3150\n",
+      "\n",
+      "epoch 3 validation\n",
+      "\n",
+      "epoch 3 val loss: 0.88008019, best val loss: 0.88555384\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5230,     0.1309\n",
+      "      task_2:     0.5570,     0.4368\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4430,     0.2772\n",
+      "\n",
+      "epoch: 4\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.895720\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5289,     0.0863\n",
+      "      task_2:     0.5221,     0.4890\n",
+      "      task_3:     0.8925,     0.4728\n",
+      "      task_4:     0.4473,     0.3173\n",
+      "\n",
+      "epoch 4 validation\n",
+      "\n",
+      "epoch 4 val loss: 0.87371731, best val loss: 0.88008019\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5230,     0.1309\n",
+      "      task_2:     0.5570,     0.4368\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4230,     0.2962\n",
+      "\n",
+      "epoch: 5\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.910037\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5339,     0.1302\n",
+      "      task_2:     0.5245,     0.4884\n",
+      "      task_3:     0.8929,     0.4717\n",
+      "      task_4:     0.4345,     0.3083\n",
+      "\n",
+      "epoch 5 validation\n",
+      "\n",
+      "epoch 5 val loss: 0.86977503, best val loss: 0.87371731\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5230,     0.1309\n",
+      "      task_2:     0.5620,     0.4047\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4230,     0.2962\n",
+      "\n",
+      "epoch: 6\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.877551\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5318,     0.1266\n",
+      "      task_2:     0.5261,     0.4906\n",
+      "      task_3:     0.8931,     0.4724\n",
+      "      task_4:     0.4357,     0.3079\n",
+      "\n",
+      "epoch 6 validation\n",
+      "\n",
+      "epoch 6 val loss: 0.86942204, best val loss: 0.86977503\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5090,     0.1407\n",
+      "      task_2:     0.5570,     0.4368\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4175,     0.2990\n",
+      "\n",
+      "epoch: 7\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.882671\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5315,     0.1289\n",
+      "      task_2:     0.5280,     0.4881\n",
+      "      task_3:     0.8933,     0.4718\n",
+      "      task_4:     0.4500,     0.3177\n",
+      "\n",
+      "epoch 7 validation\n",
+      "\n",
+      "epoch 7 val loss: 0.86656769, best val loss: 0.86942204\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5570,     0.4368\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4250,     0.3016\n",
+      "\n",
+      "epoch: 8\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.871553\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5358,     0.1263\n",
+      "      task_2:     0.5261,     0.4877\n",
+      "      task_3:     0.8933,     0.4725\n",
+      "      task_4:     0.4429,     0.3110\n",
+      "\n",
+      "epoch 8 validation\n",
+      "\n",
+      "epoch 8 val loss: 0.86422674, best val loss: 0.86656769\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5570,     0.4368\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4325,     0.3014\n",
+      "\n",
+      "epoch: 9\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.870834\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5339,     0.1255\n",
+      "      task_2:     0.5273,     0.4878\n",
+      "      task_3:     0.8933,     0.4718\n",
+      "      task_4:     0.4480,     0.3157\n",
+      "\n",
+      "epoch 9 validation\n",
+      "\n",
+      "epoch 9 val loss: 0.86480030, best val loss: 0.86422674\n",
+      "patience counter is at 1 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5570,     0.4368\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4175,     0.2990\n",
+      "\n",
+      "epoch: 10\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.861358\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5372,     0.1261\n",
+      "      task_2:     0.5307,     0.4907\n",
+      "      task_3:     0.8929,     0.4717\n",
+      "      task_4:     0.4461,     0.3132\n",
+      "\n",
+      "epoch 10 validation\n",
+      "\n",
+      "epoch 10 val loss: 0.86287446, best val loss: 0.86422674\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5570,     0.4368\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4200,     0.3008\n",
+      "\n",
+      "epoch: 11\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.879807\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5381,     0.1235\n",
+      "      task_2:     0.5328,     0.4900\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4429,     0.3093\n",
+      "\n",
+      "epoch 11 validation\n",
+      "\n",
+      "epoch 11 val loss: 0.85935990, best val loss: 0.86287446\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5570,     0.4368\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4325,     0.3014\n",
+      "\n",
+      "epoch: 12\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.875656\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5393,     0.1233\n",
+      "      task_2:     0.5368,     0.4913\n",
+      "      task_3:     0.8932,     0.4724\n",
+      "      task_4:     0.4445,     0.3115\n",
+      "\n",
+      "epoch 12 validation\n",
+      "\n",
+      "epoch 12 val loss: 0.86090763, best val loss: 0.85935990\n",
+      "patience counter is at 1 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5570,     0.4368\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4250,     0.3016\n",
+      "\n",
+      "epoch: 13\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.868936\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5398,     0.1244\n",
+      "      task_2:     0.5343,     0.4856\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4509,     0.3161\n",
+      "\n",
+      "epoch 13 validation\n",
+      "\n",
+      "epoch 13 val loss: 0.86076062, best val loss: 0.85935990\n",
+      "patience counter is at 2 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5570,     0.4368\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4340,     0.2940\n",
+      "\n",
+      "epoch: 14\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.857254\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5400,     0.1240\n",
+      "      task_2:     0.5388,     0.4872\n",
+      "      task_3:     0.8933,     0.4718\n",
+      "      task_4:     0.4527,     0.3176\n",
+      "\n",
+      "epoch 14 validation\n",
+      "\n",
+      "epoch 14 val loss: 0.85822257, best val loss: 0.85935990\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5520,     0.4615\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4415,     0.2875\n",
+      "\n",
+      "epoch: 15\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.870606\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5407,     0.1234\n",
+      "      task_2:     0.5333,     0.4840\n",
+      "      task_3:     0.8933,     0.4718\n",
+      "      task_4:     0.4442,     0.3111\n",
+      "\n",
+      "epoch 15 validation\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "epoch 15 val loss: 0.85979277, best val loss: 0.85822257\n",
+      "patience counter is at 1 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5570,     0.4368\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4325,     0.3017\n",
+      "\n",
+      "epoch: 16\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.854823\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5417,     0.1225\n",
+      "      task_2:     0.5428,     0.4932\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4465,     0.3124\n",
+      "\n",
+      "epoch 16 validation\n",
+      "\n",
+      "epoch 16 val loss: 0.86009715, best val loss: 0.85822257\n",
+      "patience counter is at 2 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5410,     0.4546\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4415,     0.2875\n",
+      "\n",
+      "epoch: 17\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.866319\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5412,     0.1232\n",
+      "      task_2:     0.5412,     0.4870\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4477,     0.3126\n",
+      "\n",
+      "epoch 17 validation\n",
+      "\n",
+      "epoch 17 val loss: 0.85758074, best val loss: 0.85822257\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5290,     0.1332\n",
+      "      task_2:     0.5410,     0.4546\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4340,     0.2940\n",
+      "\n",
+      "epoch: 18\n",
+      "\n",
+      "training time 1.49\n",
+      "Training loss: 0.859950\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5419,     0.1217\n",
+      "      task_2:     0.5407,     0.4852\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4567,     0.3192\n",
+      "\n",
+      "epoch 18 validation\n",
+      "\n",
+      "epoch 18 val loss: 0.85800012, best val loss: 0.85758074\n",
+      "patience counter is at 1 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5570,     0.4368\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4340,     0.2940\n",
+      "\n",
+      "epoch: 19\n",
+      "\n",
+      "training time 1.44\n",
+      "Training loss: 0.881447\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5432,     0.1221\n",
+      "      task_2:     0.5403,     0.4861\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4563,     0.3167\n",
+      "\n",
+      "epoch 19 validation\n",
+      "\n",
+      "epoch 19 val loss: 0.85840940, best val loss: 0.85758074\n",
+      "patience counter is at 2 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5570,     0.4368\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4340,     0.2940\n",
+      "\n",
+      "epoch: 20\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.863346\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5423,     0.1215\n",
+      "      task_2:     0.5392,     0.4842\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4571,     0.3186\n",
+      "\n",
+      "epoch 20 validation\n",
+      "\n",
+      "epoch 20 val loss: 0.85703684, best val loss: 0.85758074\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5360,     0.4729\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4340,     0.2940\n",
+      "\n",
+      "epoch: 21\n",
+      "\n",
+      "training time 1.42\n",
+      "Training loss: 0.847458\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5424,     0.1223\n",
+      "      task_2:     0.5400,     0.4816\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4557,     0.3171\n",
+      "\n",
+      "epoch 21 validation\n",
+      "\n",
+      "epoch 21 val loss: 0.85821556, best val loss: 0.85703684\n",
+      "patience counter is at 1 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5290,     0.1332\n",
+      "      task_2:     0.5410,     0.4546\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4465,     0.2757\n",
+      "\n",
+      "epoch: 22\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.858682\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5443,     0.1232\n",
+      "      task_2:     0.5521,     0.4929\n",
+      "      task_3:     0.8933,     0.4725\n",
+      "      task_4:     0.4556,     0.3143\n",
+      "\n",
+      "epoch 22 validation\n",
+      "\n",
+      "epoch 22 val loss: 0.85889643, best val loss: 0.85703684\n",
+      "patience counter is at 2 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5570,     0.4368\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4340,     0.2940\n",
+      "\n",
+      "epoch: 23\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.866145\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5434,     0.1224\n",
+      "      task_2:     0.5429,     0.4828\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4542,     0.3154\n",
+      "\n",
+      "epoch 23 validation\n",
+      "\n",
+      "epoch 23 val loss: 0.85755726, best val loss: 0.85703684\n",
+      "patience counter is at 3 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5360,     0.4729\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4390,     0.2861\n",
+      "\n",
+      "epoch: 24\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.857183\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5442,     0.1228\n",
+      "      task_2:     0.5444,     0.4811\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4546,     0.3160\n",
+      "\n",
+      "epoch 24 validation\n",
+      "\n",
+      "epoch 24 val loss: 0.85645615, best val loss: 0.85703684\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5570,     0.4368\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4290,     0.2986\n",
+      "\n",
+      "epoch: 25\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.858206\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5440,     0.1219\n",
+      "      task_2:     0.5440,     0.4837\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4466,     0.3083\n",
+      "\n",
+      "epoch 25 validation\n",
+      "\n",
+      "epoch 25 val loss: 0.85722474, best val loss: 0.85645615\n",
+      "patience counter is at 1 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5290,     0.1332\n",
+      "      task_2:     0.5570,     0.4368\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4340,     0.2940\n",
+      "\n",
+      "epoch: 26\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.863430\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5443,     0.1213\n",
+      "      task_2:     0.5419,     0.4765\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4538,     0.3139\n",
+      "\n",
+      "epoch 26 validation\n",
+      "\n",
+      "epoch 26 val loss: 0.85645027, best val loss: 0.85645615\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5440,     0.4582\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4465,     0.2757\n",
+      "\n",
+      "epoch: 27\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.844315\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5455,     0.1215\n",
+      "      task_2:     0.5433,     0.4795\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4459,     0.3082\n",
+      "\n",
+      "epoch 27 validation\n",
+      "\n",
+      "epoch 27 val loss: 0.85848679, best val loss: 0.85645027\n",
+      "patience counter is at 1 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5100,     0.1419\n",
+      "      task_2:     0.5410,     0.4546\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4395,     0.2738\n",
+      "\n",
+      "epoch: 28\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.858502\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5436,     0.1210\n",
+      "      task_2:     0.5499,     0.4883\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4592,     0.3174\n",
+      "\n",
+      "epoch 28 validation\n",
+      "\n",
+      "epoch 28 val loss: 0.85665422, best val loss: 0.85645027\n",
+      "patience counter is at 2 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5410,     0.4546\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4520,     0.2369\n",
+      "\n",
+      "epoch: 29\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.862125\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5440,     0.1203\n",
+      "      task_2:     0.5436,     0.4778\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4511,     0.3130\n",
+      "\n",
+      "epoch 29 validation\n",
+      "\n",
+      "epoch 29 val loss: 0.85837055, best val loss: 0.85645027\n",
+      "patience counter is at 3 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5410,     0.4546\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4470,     0.2592\n",
+      "\n",
+      "epoch: 30\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.844298\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5441,     0.1210\n",
+      "      task_2:     0.5428,     0.4795\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4573,     0.3168\n",
+      "\n",
+      "epoch 30 validation\n",
+      "\n",
+      "epoch 30 val loss: 0.85640588, best val loss: 0.85645027\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5410,     0.4546\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4420,     0.2751\n",
+      "\n",
+      "epoch: 31\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "training time 1.44\n",
+      "Training loss: 0.845142\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5456,     0.1204\n",
+      "      task_2:     0.5495,     0.4848\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4543,     0.3126\n",
+      "\n",
+      "epoch 31 validation\n",
+      "\n",
+      "epoch 31 val loss: 0.85564961, best val loss: 0.85640588\n",
+      "patience counter is at 0 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5280,     0.4687\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4445,     0.2579\n",
+      "\n",
+      "epoch: 32\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.845179\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5455,     0.1206\n",
+      "      task_2:     0.5508,     0.4790\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4560,     0.3145\n",
+      "\n",
+      "epoch 32 validation\n",
+      "\n",
+      "epoch 32 val loss: 0.85590241, best val loss: 0.85564961\n",
+      "patience counter is at 1 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5280,     0.4687\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4445,     0.2579\n",
+      "\n",
+      "epoch: 33\n",
+      "\n",
+      "training time 1.42\n",
+      "Training loss: 0.855577\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5441,     0.1209\n",
+      "      task_2:     0.5495,     0.4824\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4530,     0.3126\n",
+      "\n",
+      "epoch 33 validation\n",
+      "\n",
+      "epoch 33 val loss: 0.85630451, best val loss: 0.85564961\n",
+      "patience counter is at 2 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5430,     0.1173\n",
+      "      task_2:     0.5440,     0.4582\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4320,     0.2835\n",
+      "\n",
+      "epoch: 34\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.859294\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5459,     0.1202\n",
+      "      task_2:     0.5477,     0.4793\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4585,     0.3162\n",
+      "\n",
+      "epoch 34 validation\n",
+      "\n",
+      "epoch 34 val loss: 0.85754175, best val loss: 0.85564961\n",
+      "patience counter is at 3 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5290,     0.1332\n",
+      "      task_2:     0.5230,     0.4818\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4415,     0.2875\n",
+      "\n",
+      "epoch: 35\n",
+      "\n",
+      "training time 1.49\n",
+      "Training loss: 0.856460\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5449,     0.1213\n",
+      "      task_2:     0.5433,     0.4711\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4531,     0.3137\n",
+      "\n",
+      "epoch 35 validation\n",
+      "\n",
+      "epoch 35 val loss: 0.85775835, best val loss: 0.85564961\n",
+      "patience counter is at 4 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5100,     0.1419\n",
+      "      task_2:     0.5280,     0.4687\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4325,     0.3017\n",
+      "\n",
+      "epoch: 36\n",
+      "\n",
+      "training time 1.43\n",
+      "Training loss: 0.868647\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5455,     0.1201\n",
+      "      task_2:     0.5523,     0.4849\n",
+      "      task_3:     0.8935,     0.4719\n",
+      "      task_4:     0.4549,     0.3124\n",
+      "\n",
+      "epoch 36 validation\n",
+      "\n",
+      "epoch 36 val loss: 0.85662092, best val loss: 0.85564961\n",
+      "patience counter is at 5 of 5\n",
+      "        task:      micro        macro\n",
+      "      task_1:     0.5290,     0.1332\n",
+      "      task_2:     0.5440,     0.4582\n",
+      "      task_3:     0.8890,     0.4706\n",
+      "      task_4:     0.4395,     0.2738\n",
+      "saving to savedmodels/clc_test/clc_test_foldclc.h5\n",
+      "\n",
+      "Evaluating train set\n",
+      "\n",
+      "Evaluating test set\n",
+      "\n",
+      "Evaluating val set\n",
+      "Saving predictions to csv\n",
+      "\n",
+      "Full model file has been saved at savedmodels/clc_test/clc_test_20230523212643_fold0_clc_full.h5\n"
+     ]
+    }
+   ],
+   "source": [
+    "run_clc.run_case_level(args)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}


Test patch:
diff --git a/tests/model_args.yml b/tests/model_args.yml
index 8aa8c4a..6a35167 100644
--- a/tests/model_args.yml
+++ b/tests/model_args.yml
@@ -30,7 +30,7 @@ train_kwargs:
   max_epochs: 100
   patience: 5
   mixed_precision: False
-  class_weights: #  '../weights/P3B3_weights.pickle' # path, dict, or blank
+  class_weights: #  '../fresco/weights/P3B3_weights.pickle' # path, dict, or blank
   # copied from data_kwargs
   # multilabel: False
   # random_seed: 1234
diff --git a/tests/model_tests.py b/tests/model_tests.py
index 27acbee..81babc3 100644
--- a/tests/model_tests.py
+++ b/tests/model_tests.py
@@ -2,7 +2,7 @@
 import json
 import os
 import random
-import sys 
+import sys
 
 import torch
 
@@ -125,24 +125,24 @@ def test_loss_multiple(params, seed, data_loaders, dw, device, mode ):
     print('------testing loss multiple tasks')
     params.model_args['train_kwargs']['class_weights'] = None
     if mode=='train':
-        run_train_test(params, seed, data_loaders, dw, device, 4)     
+        run_train_test(params, seed, data_loaders, dw, device, 4)
     elif mode=='evaluate':
         run_eval_test(params, seed, data_loaders, dw, device, 4)
 
 def test_weighted_loss_multiple(params, seed, data_loaders, dw, device, mode ):
     print('------testing weighted loss multiple tasks')
-    params.model_args['train_kwargs']['class_weights'] = '../weights/P3B3_weights.pickle'
+    params.model_args['train_kwargs']['class_weights'] = '../fresco/weights/P3B3_weights.pickle'
     model, trainer = get_trainer(params, dw, device)
     if mode=='train':
-        run_train_test(params, seed, data_loaders, dw, device, 4)     
+        run_train_test(params, seed, data_loaders, dw, device, 4)
     elif mode=='evaluate':
         run_eval_test(params, seed, data_loaders, dw, device, 4)
 
 def test_weighted_loss_single(params, seed, data_loaders, dw, device, mode ):
     print('------testing weighted loss single task')
-    params.model_args['train_kwargs']['class_weights'] = '../weights/P3B3_weights.pickle'
+    params.model_args['train_kwargs']['class_weights'] = '../fresco/weights/P3B3_weights.pickle'
     if mode=='train':
-        run_train_test(params, seed, data_loaders, dw, device, 1)     
+        run_train_test(params, seed, data_loaders, dw, device, 1)
     elif mode=='evaluate':
         run_eval_test(params, seed, data_loaders, dw, device, 1)
 
@@ -150,22 +150,18 @@ def test_loss_single(params, seed, data_loaders, dw, device, mode ):
     print('------testing loss single task')
     params.model_args['train_kwargs']['class_weights'] = None
     if mode=='train':
-        run_train_test(params, seed, data_loaders, dw, device, 1)     
+        run_train_test(params, seed, data_loaders, dw, device, 1)
     elif mode=='evaluate':
         run_eval_test(params, seed, data_loaders, dw, device, 1)
 
 def run_train_test(params, seed, data_loaders, dw, device, nloss):
     model, trainer = get_trainer(params, dw, device)
-    assert len(trainer.loss_funs) == nloss 
-    for batch in data_loaders['train']:
-        loss = trainer.compute_loss(batch, dac=None)
-        loss.backward()
-        break
+    assert len(trainer.loss_funs) == nloss
 
 def run_eval_test(params, seed, data_loaders, dw, device, nloss):
     model, trainer = get_trainer(params, dw, device)
     evaluator = predictions.ScoreModel(params.model_args, data_loaders, model, device)
-    assert len(evaluator.loss_funs) == nloss 
+    assert len(evaluator.loss_funs) == nloss
     evaluator.evaluate_model(dw.dict_maps['id2label'], dac=None)
 
 if __name__ == '__main__':