space/gradio_tryout.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e25090fa-f990-4f1a-84f3-b12159eedae8",
   "metadata": {},
   "source": [
    "# Try out gradio"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afd23321-1870-44af-82ed-bb241d055dfa",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "3bbee2e4-55c8-4b06-9929-72026edf7932",
   "metadata": {},
   "source": [
    "**Load prerequisites**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "f8c28d2d-8458-49fd-8ebf-5e729d6e861f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Hardware accelerator e.g. GPU is available in the environment, but no `device` argument is passed to the `Pipeline` object. Model will be on CPU.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First trip: We are a couple in our thirties traveling to Vienna for a three-day city trip. We’ll be staying at a friend’s house and plan to explore the city by sightseeing, strolling through the streets, visiting markets, and trying out great restaurants and cafés. We also hope to attend a classical music concert. Our journey to Vienna will be by train. \n",
      "\n",
      "Trip type: ['city trip', ['sightseeing'], 'variable weather / spring / autumn', 'luxury (including evening wear)', 'casual', 'indoor', 'no own vehicle', 'no special condition', '3 days']\n"
     ]
    }
   ],
   "source": [
    "# Prerequisites\n",
    "from tabulate import tabulate\n",
    "from transformers import pipeline\n",
    "import json\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle\n",
    "import os\n",
    "import time\n",
    "\n",
    "# Load the model and create a pipeline for zero-shot classification (1min loading + classifying with 89 labels)\n",
    "classifier = pipeline(\"zero-shot-classification\", model=\"cross-encoder/nli-deberta-v3-base\")\n",
    "model_name = 'model_cross-encoder-nli-deberta-v3-base'\n",
    "# tried:\n",
    "# cross-encoder/nli-deberta-v3-large  gave error\n",
    "# MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli\n",
    "# facebook/bart-large-mnli\n",
    "# sileod/deberta-v3-base-tasksource-nli\n",
    "\n",
    "# get candidate labels\n",
    "with open(\"packing_label_structure.json\", \"r\") as file:\n",
    "    candidate_labels = json.load(file)\n",
    "keys_list = list(candidate_labels.keys())\n",
    "\n",
    "# Load test data (in list of dictionaries)\n",
    "with open(\"test_data.json\", \"r\") as file:\n",
    "    packing_data = json.load(file)\n",
    "# Extract all trip descriptions and trip_types\n",
    "trip_descriptions = [trip['description'] for trip in packing_data]\n",
    "trip_types = [trip['trip_types'] for trip in packing_data]\n",
    "\n",
    "# Access the first trip description\n",
    "first_trip = trip_descriptions[1]\n",
    "# Get the packing list for the secondfirst trip\n",
    "first_trip_type = trip_types[1]\n",
    "\n",
    "print(f\"First trip: {first_trip} \\n\")\n",
    "print(f\"Trip type: {first_trip_type}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "3a762755-872d-43a6-b666-874d6133488c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# function that returns pandas data frame with predictions\n",
    "\n",
    "cut_off = 0.5  # used to choose which activities are relevant\n",
    "\n",
    "def pred_trip(trip_descr, trip_type, cut_off):\n",
    "    # Create an empty DataFrame with specified columns\n",
    "    df = pd.DataFrame(columns=['superclass', 'pred_class'])\n",
    "    for i, key in enumerate(keys_list):\n",
    "        if key == 'activities':\n",
    "            result = classifier(trip_descr, candidate_labels[key], multi_label=True)\n",
    "            indices = [i for i, score in enumerate(result['scores']) if score > cut_off]\n",
    "            classes = [result['labels'][i] for i in indices]\n",
    "        else:\n",
    "            result = classifier(trip_descr, candidate_labels[key])\n",
    "            classes = result[\"labels\"][0]\n",
    "        print(result)\n",
    "        print(classes)\n",
    "        print(i)\n",
    "        df.loc[i] = [key, classes]\n",
    "    df['true_class'] = trip_type\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "3b4f3193-3bdd-453c-8664-df84f955600c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# function for accuracy, perc true classes identified and perc wrong pred classes\n",
    "\n",
    "def perf_measure(df):\n",
    "    df['same_value'] = df['pred_class'] == df['true_class']\n",
    "    correct = sum(df.loc[df.index != 1, 'same_value'])\n",
    "    total = len(df['same_value'])\n",
    "    accuracy = correct/total\n",
    "    pred_class = df.loc[df.index == 1, 'pred_class'].iloc[0]\n",
    "    true_class = df.loc[df.index == 1, 'true_class'].iloc[0]\n",
    "    correct = [label for label in pred_class if label in true_class]\n",
    "    num_correct = len(correct)\n",
    "    correct_perc = num_correct/len(true_class)\n",
    "    num_pred = len(pred_class)\n",
    "    wrong_perc = (num_pred - num_correct)/num_pred\n",
    "    df_perf = pd.DataFrame({\n",
    "    'accuracy': [accuracy],\n",
    "    'true_ident': [correct_perc],\n",
    "    'false_pred': [wrong_perc]\n",
    "    })\n",
    "    return(df_perf)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62c5c18c-58f4-465c-a188-c57cfa7ffa90",
   "metadata": {},
   "source": [
    "**Now do the same for all trips**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "4dd01755-be8d-4904-8494-ac28aba2fee7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'sequence': 'I am planning a trip to Greece with my boyfriend, where we will visit two islands. We have booked an apartment on each island for a few days and plan to spend most of our time relaxing. Our main goals are to enjoy the beach, try delicious local food, and possibly go on a hike—if it’s not too hot. We will be relying solely on public transport. We’re in our late 20s and traveling from the Netherlands.', 'labels': ['micro-adventure / weekend trip', 'digital nomad trip', 'beach vacation', 'festival trip', 'city trip', 'cultural exploration', 'road trip (car/camper)', 'camping trip (wild camping)', 'long-distance hike / thru-hike', 'hut trek (winter)', 'ski tour / skitour', 'snowboard / splitboard trip', 'nature escape', 'yoga / wellness retreat', 'hut trek (summer)', 'camping trip (campground)'], 'scores': [0.9722680449485779, 0.007802918087691069, 0.0075571718625724316, 0.0022959215566515923, 0.0021305829286575317, 0.001222927705384791, 0.0009879637509584427, 0.000805296644102782, 0.0007946204277686775, 0.0007107199053280056, 0.0007009899127297103, 0.0006353880744427443, 0.0005838185315951705, 0.0005424902774393559, 0.0004807499353773892, 0.0004804217896889895]}\n",
      "micro-adventure / weekend trip\n",
      "0\n",
      "{'sequence': 'I am planning a trip to Greece with my boyfriend, where we will visit two islands. We have booked an apartment on each island for a few days and plan to spend most of our time relaxing. Our main goals are to enjoy the beach, try delicious local food, and possibly go on a hike—if it’s not too hot. We will be relying solely on public transport. We’re in our late 20s and traveling from the Netherlands.', 'labels': ['going to the beach', 'sightseeing', 'relaxing', 'hiking', 'hut-to-hut hiking', 'stand-up paddleboarding (SUP)', 'photography', 'biking', 'running', 'ski touring', 'snowshoe hiking', 'yoga', 'kayaking / canoeing', 'horseback riding', 'rafting', 'paragliding', 'cross-country skiing', 'surfing', 'skiing', 'ice climbing', 'fishing', 'snorkeling', 'swimming', 'rock climbing', 'scuba diving'], 'scores': [0.4660525321960449, 0.007281942293047905, 0.003730606520548463, 0.0001860307966126129, 0.00014064949937164783, 0.00011034693307010457, 5.2949126256862655e-05, 3.828677654382773e-05, 3.396756437723525e-05, 1.5346524378401227e-05, 9.348185812996235e-06, 8.182429155567661e-06, 6.5973340497293975e-06, 6.271920938161202e-06, 5.544673058466287e-06, 5.299102667777333e-06, 4.855380211665761e-06, 4.506250661506783e-06, 3.949530764657538e-06, 3.730233856913401e-06, 3.297281637060223e-06, 3.0508665531669976e-06, 2.933618134193239e-06, 2.6379277642263332e-06, 2.2992651338427095e-06]}\n",
      "[]\n",
      "1\n",
      "{'sequence': 'I am planning a trip to Greece with my boyfriend, where we will visit two islands. We have booked an apartment on each island for a few days and plan to spend most of our time relaxing. Our main goals are to enjoy the beach, try delicious local food, and possibly go on a hike—if it’s not too hot. We will be relying solely on public transport. We’re in our late 20s and traveling from the Netherlands.', 'labels': ['variable weather / spring / autumn', 'warm destination / summer', 'cold destination / winter', 'dry / desert-like', 'tropical / humid'], 'scores': [0.5934922695159912, 0.17430798709392548, 0.10943299531936646, 0.07068652659654617, 0.05208020657300949]}\n",
      "variable weather / spring / autumn\n",
      "2\n",
      "{'sequence': 'I am planning a trip to Greece with my boyfriend, where we will visit two islands. We have booked an apartment on each island for a few days and plan to spend most of our time relaxing. Our main goals are to enjoy the beach, try delicious local food, and possibly go on a hike—if it’s not too hot. We will be relying solely on public transport. We’re in our late 20s and traveling from the Netherlands.', 'labels': ['minimalist', 'ultralight', 'luxury (including evening wear)', 'lightweight (but comfortable)'], 'scores': [0.6965053081512451, 0.11270010471343994, 0.10676420480012894, 0.08403033763170242]}\n",
      "minimalist\n",
      "3\n",
      "{'sequence': 'I am planning a trip to Greece with my boyfriend, where we will visit two islands. We have booked an apartment on each island for a few days and plan to spend most of our time relaxing. Our main goals are to enjoy the beach, try delicious local food, and possibly go on a hike—if it’s not too hot. We will be relying solely on public transport. We’re in our late 20s and traveling from the Netherlands.', 'labels': ['casual', 'formal (business trip)', 'conservative'], 'scores': [0.6362482309341431, 0.22082458436489105, 0.14292724430561066]}\n",
      "casual\n",
      "4\n",
      "{'sequence': 'I am planning a trip to Greece with my boyfriend, where we will visit two islands. We have booked an apartment on each island for a few days and plan to spend most of our time relaxing. Our main goals are to enjoy the beach, try delicious local food, and possibly go on a hike—if it’s not too hot. We will be relying solely on public transport. We’re in our late 20s and traveling from the Netherlands.', 'labels': ['indoor', 'sleeping in a tent', 'huts with half board', 'sleeping in a car'], 'scores': [0.435793399810791, 0.20242486894130707, 0.19281964004039764, 0.16896207630634308]}\n",
      "indoor\n",
      "5\n",
      "{'sequence': 'I am planning a trip to Greece with my boyfriend, where we will visit two islands. We have booked an apartment on each island for a few days and plan to spend most of our time relaxing. Our main goals are to enjoy the beach, try delicious local food, and possibly go on a hike—if it’s not too hot. We will be relying solely on public transport. We’re in our late 20s and traveling from the Netherlands.', 'labels': ['no own vehicle', 'own vehicle'], 'scores': [0.9987181425094604, 0.0012818538816645741]}\n",
      "no own vehicle\n",
      "6\n",
      "{'sequence': 'I am planning a trip to Greece with my boyfriend, where we will visit two islands. We have booked an apartment on each island for a few days and plan to spend most of our time relaxing. Our main goals are to enjoy the beach, try delicious local food, and possibly go on a hike—if it’s not too hot. We will be relying solely on public transport. We’re in our late 20s and traveling from the Netherlands.', 'labels': ['self-supported (bring your own food/cooking)', 'no special conditions', 'off-grid / no electricity', 'rainy climate', 'child-friendly', 'snow and ice', 'pet-friendly', 'high alpine terrain', 'avalanche-prone terrain'], 'scores': [0.1984991431236267, 0.1695038080215454, 0.16221018135547638, 0.13200421631336212, 0.12101645022630692, 0.10550825297832489, 0.042406272143125534, 0.03797775134444237, 0.030873913317918777]}\n",
      "self-supported (bring your own food/cooking)\n",
      "7\n",
      "{'sequence': 'I am planning a trip to Greece with my boyfriend, where we will visit two islands. We have booked an apartment on each island for a few days and plan to spend most of our time relaxing. Our main goals are to enjoy the beach, try delicious local food, and possibly go on a hike—if it’s not too hot. We will be relying solely on public transport. We’re in our late 20s and traveling from the Netherlands.', 'labels': ['7+ days', '2 days', '1 day', '7 days', '5 days', '3 days', '6 days', '4 days'], 'scores': [0.4730822443962097, 0.1168912723660469, 0.10058756172657013, 0.0991850346326828, 0.05424537882208824, 0.053677864372730255, 0.051554784178733826, 0.050775907933712006]}\n",
      "7+ days\n",
      "8\n",
      "           superclass                                    pred_class  \\\n",
      "0       activity_type                micro-adventure / weekend trip   \n",
      "1          activities                                            []   \n",
      "2   climate_or_season            variable weather / spring / autumn   \n",
      "3    style_or_comfort                                    minimalist   \n",
      "4          dress_code                                        casual   \n",
      "5       accommodation                                        indoor   \n",
      "6      transportation                                no own vehicle   \n",
      "7  special_conditions  self-supported (bring your own food/cooking)   \n",
      "8    trip_length_days                                       7+ days   \n",
      "\n",
      "                                         true_class  \n",
      "0                                    beach vacation  \n",
      "1  [swimming, going to the beach, relaxing, hiking]  \n",
      "2                         warm destination / summer  \n",
      "3                     lightweight (but comfortable)  \n",
      "4                                            casual  \n",
      "5                                            indoor  \n",
      "6                                    no own vehicle  \n",
      "7                             no special conditions  \n",
      "8                                           7+ days  \n"
     ]
    },
    {
     "ename": "ZeroDivisionError",
     "evalue": "division by zero",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mZeroDivisionError\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[60], line 13\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[38;5;28mprint\u001b[39m(df)\n\u001b[1;32m     12\u001b[0m \u001b[38;5;66;03m# accuracy, perc true classes identified and perc wrong pred classes\u001b[39;00m\n\u001b[0;32m---> 13\u001b[0m performance \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mconcat([performance, \u001b[43mperf_measure\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdf\u001b[49m\u001b[43m)\u001b[49m])\n\u001b[1;32m     14\u001b[0m \u001b[38;5;28mprint\u001b[39m(performance)\n\u001b[1;32m     16\u001b[0m result_list\u001b[38;5;241m.\u001b[39mappend(df)\n",
      "Cell \u001b[0;32mIn[59], line 14\u001b[0m, in \u001b[0;36mperf_measure\u001b[0;34m(df)\u001b[0m\n\u001b[1;32m     12\u001b[0m correct_perc \u001b[38;5;241m=\u001b[39m num_correct\u001b[38;5;241m/\u001b[39m\u001b[38;5;28mlen\u001b[39m(true_class)\n\u001b[1;32m     13\u001b[0m num_pred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(pred_class)\n\u001b[0;32m---> 14\u001b[0m wrong_perc \u001b[38;5;241m=\u001b[39m \u001b[43m(\u001b[49m\u001b[43mnum_pred\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mnum_correct\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m/\u001b[39;49m\u001b[43mnum_pred\u001b[49m\n\u001b[1;32m     15\u001b[0m df_perf \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mDataFrame({\n\u001b[1;32m     16\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124maccuracy\u001b[39m\u001b[38;5;124m'\u001b[39m: [accuracy],\n\u001b[1;32m     17\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtrue_ident\u001b[39m\u001b[38;5;124m'\u001b[39m: [correct_perc],\n\u001b[1;32m     18\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfalse_pred\u001b[39m\u001b[38;5;124m'\u001b[39m: [wrong_perc]\n\u001b[1;32m     19\u001b[0m })\n\u001b[1;32m     20\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m(df_perf)\n",
      "\u001b[0;31mZeroDivisionError\u001b[0m: division by zero"
     ]
    }
   ],
   "source": [
    "result_list = []\n",
    "performance = pd.DataFrame(columns=['accuracy', 'true_ident', 'false_pred'])\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "for i in range(len(trip_descriptions)):\n",
    "    current_trip = trip_descriptions[i]\n",
    "    current_type = trip_types[i]\n",
    "    df = pred_trip(current_trip, current_type, cut_off = 0.5)\n",
    "    print(df)\n",
    "    \n",
    "    # accuracy, perc true classes identified and perc wrong pred classes\n",
    "    performance = pd.concat([performance, perf_measure(df)])\n",
    "    print(performance)\n",
    "    \n",
    "    result_list.append(df)\n",
    "\n",
    "end_time = time.time()\n",
    "\n",
    "elapsed_time = end_time - start_time"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5c08703-7166-4d03-9d6b-ee2c12608134",
   "metadata": {},
   "source": [
    "**Compute average performance measures**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "eb33fd31-94e6-40b5-9c36-a32effe77c01",
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[61], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# Extract \"same_value\" column from each DataFrame\u001b[39;00m\n\u001b[1;32m      2\u001b[0m sv_columns \u001b[38;5;241m=\u001b[39m [df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msame_value\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;28;01mfor\u001b[39;00m df \u001b[38;5;129;01min\u001b[39;00m result_list]  \u001b[38;5;66;03m# 'same' needs to be changed\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m sv_columns\u001b[38;5;241m.\u001b[39minsert(\u001b[38;5;241m0\u001b[39m, \u001b[43mresult_list\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msuperclass\u001b[39m\u001b[38;5;124m'\u001b[39m])\n\u001b[1;32m      5\u001b[0m \u001b[38;5;66;03m# Combine into a new DataFrame (columns side-by-side)\u001b[39;00m\n\u001b[1;32m      6\u001b[0m sv_df \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mconcat(sv_columns, axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\n",
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "# Extract \"same_value\" column from each DataFrame\n",
    "sv_columns = [df['same_value'] for df in result_list]  # 'same' needs to be changed\n",
    "sv_columns.insert(0, result_list[0]['superclass'])\n",
    "\n",
    "# Combine into a new DataFrame (columns side-by-side)\n",
    "sv_df = pd.concat(sv_columns, axis=1)\n",
    "\n",
    "print(sv_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "bf7546cb-79ce-49ad-8cee-54d02239220c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           superclass  accuracy\n",
      "0       activity_type       0.8\n",
      "1          activities       0.0\n",
      "2   climate_or_season       0.5\n",
      "3    style_or_comfort       0.3\n",
      "4          dress_code       0.8\n",
      "5       accommodation       0.8\n",
      "6      transportation       0.7\n",
      "7  special_conditions       0.2\n",
      "8    trip_length_days       0.6\n"
     ]
    }
   ],
   "source": [
    "# Compute accuracy per superclass (row means of same_value matrix excluding the first column)\n",
    "row_means = sv_df.iloc[:, 1:].mean(axis=1)\n",
    "\n",
    "df_row_means = pd.DataFrame({\n",
    "    'superclass': sv_df['superclass'],\n",
    "    'accuracy': row_means\n",
    "})\n",
    "\n",
    "print(df_row_means)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd232953-59e8-4f28-9ce8-11515a2c310b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Compute performance measures per trip (mean for each column of performance table)\n",
    "column_means = performance.mean()\n",
    "print(column_means)\n",
    "\n",
    "# Plot histograms for all numeric columns\n",
    "performance.hist(bins=10, figsize=(10, 6))\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd682c84-3eb1-4a8d-9621-b741e98e4537",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save results\n",
    "# Structure to save\n",
    "model_result = {\n",
    "    'model': model_name,\n",
    "    'predictions': result_list,\n",
    "    'performance': performance,\n",
    "    'perf_summary': column_means,\n",
    "    'perf_superclass': df_row_means,\n",
    "    'elapsed_time': elapsed_time\n",
    "}\n",
    "\n",
    "# File path with folder\n",
    "filename = os.path.join('results', f'{model_name}_results.pkl')\n",
    "\n",
    "# Save the object\n",
    "with open(filename, 'wb') as f:\n",
    "    pickle.dump(model_result, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f38d0924-30b6-43cd-9bfc-fe5b0dc80411",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(elapsed_time/60)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1cbb54e-abe6-49b6-957e-0683196f3199",
   "metadata": {},
   "source": [
    "**Load and compare results**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "62ca82b0-6909-4e6c-9d2c-fed87971e5b6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: model_MoritzLaurer-DeBERTa-v3-base-mnli-fever-anli\n",
      "Performance Summary:\n",
      "accuracy      0.522222\n",
      "true_ident    0.841667\n",
      "false_pred    0.572381\n",
      "dtype: float64\n",
      "----------------------------------------\n",
      "Model: model_a_facebook-bart-large-mnli\n",
      "Performance Summary:\n",
      "accuracy      0.454545\n",
      "true_ident    0.689394\n",
      "false_pred    0.409091\n",
      "dtype: float64\n",
      "----------------------------------------\n",
      "Model: model_b_sileod-deberta-v3-base-tasksource-nli\n",
      "Performance Summary:\n",
      "accuracy      0.500000\n",
      "true_ident    0.666667\n",
      "false_pred    0.551667\n",
      "dtype: float64\n",
      "----------------------------------------\n",
      "Model: model_MoritzLaurer-DeBERTa-v3-base-mnli-fever-anli\n",
      "Performance Summary:\n",
      "           superclass  accuracy\n",
      "0       activity_type       0.8\n",
      "1          activities       0.0\n",
      "2   climate_or_season       0.5\n",
      "3    style_or_comfort       0.3\n",
      "4          dress_code       0.8\n",
      "5       accommodation       0.8\n",
      "6      transportation       0.7\n",
      "7  special_conditions       0.2\n",
      "8    trip_length_days       0.6\n",
      "----------------------------------------\n",
      "Model: model_a_facebook-bart-large-mnli\n",
      "Performance Summary:\n",
      "           superclass  accuracy\n",
      "0       activity_type       0.8\n",
      "1          activities       0.0\n",
      "2   climate_or_season       0.6\n",
      "3    style_or_comfort       0.4\n",
      "4          dress_code       0.7\n",
      "5       accommodation       0.3\n",
      "6      transportation       0.8\n",
      "7  special_conditions       0.0\n",
      "8    trip_length_days       0.5\n",
      "----------------------------------------\n",
      "Model: model_b_sileod-deberta-v3-base-tasksource-nli\n",
      "Performance Summary:\n",
      "           superclass  accuracy\n",
      "0       activity_type       0.7\n",
      "1          activities       0.1\n",
      "2   climate_or_season       0.6\n",
      "3    style_or_comfort       0.4\n",
      "4          dress_code       0.6\n",
      "5       accommodation       0.9\n",
      "6      transportation       0.7\n",
      "7  special_conditions       0.1\n",
      "8    trip_length_days       0.5\n",
      "----------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# Folder where your .pkl files are saved\n",
    "results_dir = 'results'\n",
    "\n",
    "# Dictionary to store all loaded results\n",
    "all_results = {}\n",
    "\n",
    "# Loop through all .pkl files in the folder\n",
    "for filename in os.listdir(results_dir):\n",
    "    if filename.endswith('.pkl'):\n",
    "        model_name = filename.replace('_results.pkl', '')  # Extract model name\n",
    "        file_path = os.path.join(results_dir, filename)\n",
    "        \n",
    "        # Load the result\n",
    "        with open(file_path, 'rb') as f:\n",
    "            result = pickle.load(f)\n",
    "            all_results[model_name] = result\n",
    "\n",
    "# Compare performance across models\n",
    "for model, data in all_results.items():\n",
    "    print(f\"Model: {model}\")\n",
    "    print(f\"Performance Summary:\\n{data['perf_summary']}\")\n",
    "    print(\"-\" * 40)\n",
    "\n",
    "\n",
    "# Compare performance across models\n",
    "for model, data in all_results.items():\n",
    "    print(f\"Model: {model}\")\n",
    "    print(f\"Performance Summary:\\n{data['perf_superclass']}\")\n",
    "    print(\"-\" * 40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "57fd150d-1cda-4be5-806b-ef380469243a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: model_MoritzLaurer-DeBERTa-v3-base-mnli-fever-anli\n",
      "Time in minutes for 10 trips:\n",
      "83.45150986512502\n",
      "----------------------------------------\n",
      "Model: model_a_facebook-bart-large-mnli\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "'elapsed_time'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[69], line 4\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m model, data \u001b[38;5;129;01min\u001b[39;00m all_results\u001b[38;5;241m.\u001b[39mitems():\n\u001b[1;32m      3\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mModel: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmodel\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 4\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTime in minutes for 10 trips:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00mdata[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124melapsed_time\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m/\u001b[39m\u001b[38;5;241m60\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m      5\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m-\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m*\u001b[39m \u001b[38;5;241m40\u001b[39m)\n",
      "\u001b[0;31mKeyError\u001b[0m: 'elapsed_time'"
     ]
    }
   ],
   "source": [
    "# Compare across models\n",
    "for model, data in all_results.items():\n",
    "    print(f\"Model: {model}\")\n",
    "    print(f\"Time in minutes for 10 trips:\\n{data['elapsed_time']/60}\")\n",
    "    print(\"-\" * 40)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17483df4-55c4-41cd-b8a9-61f7a5c7e8a3",
   "metadata": {},
   "source": [
    "**Use gradio for user input**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "cb7fd425-d0d6-458d-97ca-2150dc55f206",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running on local URL:  http://127.0.0.1:7861\n",
      "Running on public URL: https://aa06d5d85ffadaa92b.gradio.live\n",
      "\n",
      "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><iframe src=\"https://aa06d5d85ffadaa92b.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n"
     ]
    }
   ],
   "source": [
    "# use model with gradio\n",
    "from transformers import pipeline\n",
    "import gradio as gr\n",
    "\n",
    "# make a function for what I am doing\n",
    "def classify(text):\n",
    "    df = pd.DataFrame(columns=['Superclass', 'class'])\n",
    "    for i, key in enumerate(keys_list):\n",
    "        # Run the classification (ca 30 seconds classifying)\n",
    "        if key == 'activities':\n",
    "            result = classifier(text, candidate_labels[key], multi_label=True)\n",
    "            classes = [result['labels'][i] for i in indices]\n",
    "        else:\n",
    "            result = classifier(text, candidate_labels[key])\n",
    "            classes = result[\"labels\"][0]\n",
    "        print(i)\n",
    "        df.loc[i] = [key, classes]\n",
    "\n",
    "    return df\n",
    "\n",
    "demo = gr.Interface(\n",
    "    fn=classify,\n",
    "    inputs=\"text\",\n",
    "    outputs=\"dataframe\",\n",
    "    title=\"Zero-Shot Classification\",\n",
    "    description=\"Enter a text describing your trip\",\n",
    ")\n",
    "\n",
    "# Launch the Gradio app\n",
    "if __name__ == \"__main__\":\n",
    "    demo.launch(share=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e856a9c-a66c-4c4b-b7cf-8c52abbbc6fa",
   "metadata": {},
   "source": [
    "Use model with gradio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "521d9118-b59d-4cc6-b637-20202eaf8f33",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running on local URL:  http://127.0.0.1:7861\n",
      "Running on public URL: https://0f70ba5369d721cf8f.gradio.live\n",
      "\n",
      "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><iframe src=\"https://0f70ba5369d721cf8f.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Define the Gradio interface\n",
    "def classify(text):\n",
    "    return classifier(text, class_labels)\n",
    "\n",
    "demo = gr.Interface(\n",
    "    fn=classify,\n",
    "    inputs=\"text\",\n",
    "    outputs=\"json\",\n",
    "    title=\"Zero-Shot Classification\",\n",
    "    description=\"Enter a text describing your trip\",\n",
    ")\n",
    "\n",
    "# Launch the Gradio app\n",
    "if __name__ == \"__main__\":\n",
    "    demo.launch(share=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8da1c90-d3a3-4b08-801c-b3afa17b2633",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (huggingface_env)",
   "language": "python",
   "name": "huggingface_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.20"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}