Added deps

notebooks/Results LSTM.ipynb

@@ -0,0 +1,461 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Toxic Spans Detection"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "\n",
+    "import torch\n",
+    "import torch.nn as nn\n",
+    "\n",
+    "import spacy\n",
+    "import ast\n",
+    "from termcolor import colored\n",
+    "\n",
+    "from tqdm import tqdm\n",
+    "import gdown\n",
+    "\n",
+    "from utils.processing import get_index_toxic_words, color_toxic_words, f1\n",
+    "from utils.lstm import spacy_tokenizer, get_vocab\n",
+    "\n",
+    "sns.set_style('darkgrid')\n",
+    "dev = 'cuda:0' if torch.cuda.is_available() else 'cpu'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# To plot using LaTeX, sometimes it gives trouble, in that case comment these two lines\n",
+    "plt.rc('text', usetex=True)\n",
+    "plt.rc('font', family='serif')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Resultados\n",
+    "\n",
+    "De los distintos preprocesamientos el que mayor F1 score tiene en `test` es poner los posts con [ ] como completamente tóxicos (**best-model-try2.pt**, *train*=0.6498 , *test*=0.6526), así que usaremos ese."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train = pd.read_csv('../data/tsd_train.csv', converters={'spans':ast.literal_eval})\n",
+    "test = pd.read_csv('../data/tsd_trial.csv', converters={'spans':ast.literal_eval})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Nuestros embeddings\n",
+    "vocab = get_vocab(train)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class LSTMTagger(nn.Module):\n",
+    "\n",
+    "    def __init__(self, embedding_dim, stacked_layers, dropout_p, weight, hidden_dim, vocab_size):\n",
+    "        super(LSTMTagger, self).__init__()\n",
+    "        self.hidden_dim = hidden_dim         # Dimension del estado oculta en cada direccion de la LSTM\n",
+    "        self.stacked_layers = stacked_layers # Cuantas capas en la LSTM\n",
+    "        \n",
+    "        self.word_embeddings = nn.Embedding.from_pretrained(weight)\n",
+    "        self.lstm = nn.LSTM(embedding_dim,\n",
+    "                            hidden_dim,\n",
+    "                            num_layers=stacked_layers,\n",
+    "                            dropout=dropout_p,\n",
+    "                            bidirectional=True)\n",
+    "\n",
+    "        # Linear layers\n",
+    "        self.fc1 = nn.Linear(hidden_dim*2, 1) # 2 veces el tamaño de hidden_dim por ser bidireccional\n",
+    "\n",
+    "    def forward(self, sentence):\n",
+    "        embeds = self.word_embeddings(sentence)\n",
+    "        output, _ = self.lstm(embeds.view(len(sentence), 1, -1))\n",
+    "        x = torch.sigmoid(self.fc1(output.view(len(sentence), -1)))\n",
+    "        return x"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def prepare_sequence(seq):\n",
+    "    idxs = vocab.lookup_indices(seq)      # Si no está lo pone como 0\n",
+    "    return torch.tensor(idxs, dtype=torch.long, device=dev)\n",
+    "\n",
+    "def prepare_sequence_tags(seq):\n",
+    "    tag_to_ix = {\"non_toxic\": 0, \"toxic\": 1} \n",
+    "    idxs = [tag_to_ix[s] for s in seq]\n",
+    "    return torch.tensor(idxs, dtype=torch.long, device=dev)\n",
+    "\n",
+    "def tagger_LSTM(text, threshold=0.5):\n",
+    "    \"\"\"\n",
+    "    Hace el tagging con el modelo que entrenamos.\n",
+    "    \"\"\"\n",
+    "    ix_to_tag = {0: 'non_toxic', 1: 'toxic'}\n",
+    "    words = spacy_tokenizer(text.lower()) # Parece funcionar mejor\n",
+    "    \n",
+    "    with torch.no_grad():\n",
+    "        inputs = prepare_sequence(words)\n",
+    "        tag_scores = model(inputs)\n",
+    "        \n",
+    "        tags = [1 if x > threshold else 0 for x in tag_scores]\n",
+    "        tagged_sentence = list(zip(words, tags))\n",
+    "\n",
+    "    return tagged_sentence"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "El archivo con el modelo está algo pesado para GitHub, así que lo pondremos en un Drive para descargarlo."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "url = 'https://drive.google.com/uc?id=1KO-QXUBfwzjauWLhiVi9StD3y0GtiBbj'\n",
+    "output = 'models/best-model.pt'\n",
+    "\n",
+    "gdown.download(url, output, quiet=False) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "LSTMTagger(\n",
+       "  (word_embeddings): Embedding(19611, 200)\n",
+       "  (lstm): LSTM(200, 600, num_layers=6, dropout=0.2, bidirectional=True)\n",
+       "  (fc1): Linear(in_features=1200, out_features=1, bias=True)\n",
+       ")"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model = torch.load(output)\n",
+    "model.to(torch.device(dev))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0\u001b[1m\u001b[36m Pred: \u001b[0mB\u001b[0me\u001b[0mc\u001b[0ma\u001b[0mu\u001b[0ms\u001b[0me\u001b[0m \u001b[0mh\u001b[0me\u001b[0m'\u001b[0ms\u001b[0m \u001b[0ma\u001b[0m \u001b[0m\u001b[41mm\u001b[0m\u001b[41mo\u001b[0m\u001b[41mr\u001b[0m\u001b[41mo\u001b[0m\u001b[41mn\u001b[0m \u001b[0ma\u001b[0mn\u001b[0md\u001b[0m \u001b[0ma\u001b[0m \u001b[0m\u001b[41mb\u001b[0m\u001b[41mi\u001b[0m\u001b[41mg\u001b[0m\u001b[41mo\u001b[0m\u001b[41mt\u001b[0m.\u001b[0m \u001b[0mI\u001b[0mt\u001b[0m'\u001b[0ms\u001b[0m \u001b[0mn\u001b[0mo\u001b[0mt\u001b[0m \u001b[0ma\u001b[0mn\u001b[0my\u001b[0m \u001b[0mm\u001b[0mo\u001b[0mr\u001b[0me\u001b[0m \u001b[0mc\u001b[0mo\u001b[0mm\u001b[0mp\u001b[0ml\u001b[0mi\u001b[0mc\u001b[0ma\u001b[0mt\u001b[0me\u001b[0md\u001b[0m \u001b[0mt\u001b[0mh\u001b[0ma\u001b[0mn\u001b[0m \u001b[0mt\u001b[0mh\u001b[0ma\u001b[0mt\u001b[0m.\u001b[0m\n",
+      "0\u001b[1m\u001b[33m Gold: \u001b[0mB\u001b[0me\u001b[0mc\u001b[0ma\u001b[0mu\u001b[0ms\u001b[0me\u001b[0m \u001b[0mh\u001b[0me\u001b[0m'\u001b[0ms\u001b[0m \u001b[0ma\u001b[0m \u001b[0m\u001b[41mm\u001b[0m\u001b[41mo\u001b[0m\u001b[41mr\u001b[0m\u001b[41mo\u001b[0m\u001b[41mn\u001b[0m \u001b[0ma\u001b[0mn\u001b[0md\u001b[0m \u001b[0ma\u001b[0m \u001b[0m\u001b[41mb\u001b[0m\u001b[41mi\u001b[0m\u001b[41mg\u001b[0m\u001b[41mo\u001b[0m\u001b[41mt\u001b[0m.\u001b[0m \u001b[0mI\u001b[0mt\u001b[0m'\u001b[0ms\u001b[0m \u001b[0mn\u001b[0mo\u001b[0mt\u001b[0m \u001b[0ma\u001b[0mn\u001b[0my\u001b[0m \u001b[0mm\u001b[0mo\u001b[0mr\u001b[0me\u001b[0m \u001b[0mc\u001b[0mo\u001b[0mm\u001b[0mp\u001b[0ml\u001b[0mi\u001b[0mc\u001b[0ma\u001b[0mt\u001b[0me\u001b[0md\u001b[0m \u001b[0mt\u001b[0mh\u001b[0ma\u001b[0mn\u001b[0m \u001b[0mt\u001b[0mh\u001b[0ma\u001b[0mt\u001b[0m.\u001b[0m\n",
+      "\n",
+      "\n",
+      "1\u001b[1m\u001b[36m Pred: \u001b[0mH\u001b[0mo\u001b[0mw\u001b[0m \u001b[0ma\u001b[0mb\u001b[0mo\u001b[0mu\u001b[0mt\u001b[0m \u001b[0mw\u001b[0me\u001b[0m \u001b[0ms\u001b[0mt\u001b[0mo\u001b[0mp\u001b[0m \u001b[0mp\u001b[0mr\u001b[0mo\u001b[0mt\u001b[0me\u001b[0mc\u001b[0mt\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m \u001b[0m\u001b[41mi\u001b[0m\u001b[41md\u001b[0m\u001b[41mi\u001b[0m\u001b[41mo\u001b[0m\u001b[41mt\u001b[0m\u001b[41ms\u001b[0m \u001b[0ma\u001b[0mn\u001b[0md\u001b[0m \u001b[0ml\u001b[0me\u001b[0mt\u001b[0m \u001b[0mn\u001b[0ma\u001b[0mt\u001b[0mu\u001b[0mr\u001b[0me\u001b[0m \u001b[0ma\u001b[0md\u001b[0md\u001b[0m \u001b[0ms\u001b[0mo\u001b[0mm\u001b[0me\u001b[0m \u001b[0mb\u001b[0ml\u001b[0me\u001b[0ma\u001b[0mc\u001b[0mh\u001b[0m \u001b[0mt\u001b[0mo\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0mg\u001b[0me\u001b[0mn\u001b[0me\u001b[0m \u001b[0mp\u001b[0mo\u001b[0mo\u001b[0ml\u001b[0m.\u001b[0m \u001b[0mW\u001b[0me\u001b[0m \u001b[0mc\u001b[0ma\u001b[0mn\u001b[0m \u001b[0ma\u001b[0ml\u001b[0mw\u001b[0ma\u001b[0my\u001b[0ms\u001b[0m \u001b[0ms\u001b[0mu\u001b[0mb\u001b[0mm\u001b[0mi\u001b[0mt\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0mi\u001b[0mr\u001b[0m \u001b[0mn\u001b[0ma\u001b[0mm\u001b[0me\u001b[0ms\u001b[0m \u001b[0mf\u001b[0mo\u001b[0mr\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0mD\u001b[0ma\u001b[0mr\u001b[0mw\u001b[0mi\u001b[0mn\u001b[0m \u001b[0ma\u001b[0mw\u001b[0ma\u001b[0mr\u001b[0md\u001b[0ms\u001b[0m.\u001b[0m\n",
+      "1\u001b[1m\u001b[33m Gold: \u001b[0mH\u001b[0mo\u001b[0mw\u001b[0m \u001b[0ma\u001b[0mb\u001b[0mo\u001b[0mu\u001b[0mt\u001b[0m \u001b[0mw\u001b[0me\u001b[0m \u001b[0ms\u001b[0mt\u001b[0mo\u001b[0mp\u001b[0m \u001b[0mp\u001b[0mr\u001b[0mo\u001b[0mt\u001b[0me\u001b[0mc\u001b[0mt\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m \u001b[0m\u001b[41mi\u001b[0m\u001b[41md\u001b[0m\u001b[41mi\u001b[0m\u001b[41mo\u001b[0m\u001b[41mt\u001b[0m\u001b[41ms\u001b[0m \u001b[0ma\u001b[0mn\u001b[0md\u001b[0m \u001b[0ml\u001b[0me\u001b[0mt\u001b[0m \u001b[0mn\u001b[0ma\u001b[0mt\u001b[0mu\u001b[0mr\u001b[0me\u001b[0m \u001b[0ma\u001b[0md\u001b[0md\u001b[0m \u001b[0ms\u001b[0mo\u001b[0mm\u001b[0me\u001b[0m \u001b[0mb\u001b[0ml\u001b[0me\u001b[0ma\u001b[0mc\u001b[0mh\u001b[0m \u001b[0mt\u001b[0mo\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0mg\u001b[0me\u001b[0mn\u001b[0me\u001b[0m \u001b[0mp\u001b[0mo\u001b[0mo\u001b[0ml\u001b[0m.\u001b[0m \u001b[0mW\u001b[0me\u001b[0m \u001b[0mc\u001b[0ma\u001b[0mn\u001b[0m \u001b[0ma\u001b[0ml\u001b[0mw\u001b[0ma\u001b[0my\u001b[0ms\u001b[0m \u001b[0ms\u001b[0mu\u001b[0mb\u001b[0mm\u001b[0mi\u001b[0mt\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0mi\u001b[0mr\u001b[0m \u001b[0mn\u001b[0ma\u001b[0mm\u001b[0me\u001b[0ms\u001b[0m \u001b[0mf\u001b[0mo\u001b[0mr\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0mD\u001b[0ma\u001b[0mr\u001b[0mw\u001b[0mi\u001b[0mn\u001b[0m \u001b[0ma\u001b[0mw\u001b[0ma\u001b[0mr\u001b[0md\u001b[0ms\u001b[0m.\u001b[0m\n",
+      "\n",
+      "\n",
+      "2\u001b[1m\u001b[36m Pred: \u001b[0mI\u001b[0mf\u001b[0m \u001b[0mp\u001b[0me\u001b[0mo\u001b[0mp\u001b[0ml\u001b[0me\u001b[0m \u001b[0m \u001b[0mw\u001b[0me\u001b[0mr\u001b[0me\u001b[0m \u001b[0m \u001b[0ms\u001b[0mm\u001b[0ma\u001b[0mr\u001b[0mt\u001b[0m,\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0my\u001b[0m \u001b[0mw\u001b[0mo\u001b[0mu\u001b[0ml\u001b[0md\u001b[0m \u001b[0m \u001b[0mB\u001b[0mo\u001b[0my\u001b[0mc\u001b[0mo\u001b[0mt\u001b[0mt\u001b[0m \u001b[0mt\u001b[0mh\u001b[0mi\u001b[0ms\u001b[0m \u001b[0m \u001b[0mi\u001b[0mn\u001b[0me\u001b[0mp\u001b[0mt\u001b[0m \u001b[0m \u001b[0ma\u001b[0mi\u001b[0mr\u001b[0ml\u001b[0mi\u001b[0mn\u001b[0me\u001b[0m,\u001b[0m \u001b[0m \u001b[0mb\u001b[0mu\u001b[0mt\u001b[0m \u001b[0m \u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0my\u001b[0m \u001b[0m \u001b[0ma\u001b[0mr\u001b[0me\u001b[0m \u001b[0m \u001b[0mn\u001b[0mo\u001b[0mt\u001b[0m \u001b[0m \u001b[0ms\u001b[0mm\u001b[0ma\u001b[0mr\u001b[0mt\u001b[0m,\u001b[0m \u001b[0m \u001b[0ms\u001b[0mo\u001b[0m \u001b[0m \u001b[0m \u001b[0mr\u001b[0mo\u001b[0mg\u001b[0mu\u001b[0me\u001b[0m \u001b[0m \u001b[0mb\u001b[0mu\u001b[0ms\u001b[0mi\u001b[0mn\u001b[0me\u001b[0ms\u001b[0ms\u001b[0me\u001b[0ms\u001b[0m \u001b[0m \u001b[0ml\u001b[0mi\u001b[0mk\u001b[0me\u001b[0m \u001b[0m \u001b[0mt\u001b[0mh\u001b[0mi\u001b[0ms\u001b[0m \u001b[0m \u001b[0mo\u001b[0mn\u001b[0me\u001b[0m,\u001b[0m \u001b[0m \u001b[0m \u001b[0ms\u001b[0mt\u001b[0mi\u001b[0ml\u001b[0ml\u001b[0m \u001b[0mt\u001b[0mh\u001b[0mr\u001b[0mi\u001b[0mv\u001b[0me\u001b[0m \u001b[0m \u001b[0m \u001b[0mt\u001b[0ma\u001b[0mk\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0m\u001b[41mi\u001b[0m\u001b[41md\u001b[0m\u001b[41mi\u001b[0m\u001b[41mo\u001b[0m\u001b[41mt\u001b[0m\u001b[41ms\u001b[0m \u001b[0m \u001b[0mf\u001b[0mo\u001b[0mr\u001b[0m \u001b[0m \u001b[0ma\u001b[0m \u001b[0mr\u001b[0mi\u001b[0md\u001b[0me\u001b[0m.\u001b[0m.\u001b[0m.\u001b[0m\n",
+      "2\u001b[1m\u001b[33m Gold: \u001b[0mI\u001b[0mf\u001b[0m \u001b[0mp\u001b[0me\u001b[0mo\u001b[0mp\u001b[0ml\u001b[0me\u001b[0m \u001b[0m \u001b[0mw\u001b[0me\u001b[0mr\u001b[0me\u001b[0m \u001b[0m \u001b[0ms\u001b[0mm\u001b[0ma\u001b[0mr\u001b[0mt\u001b[0m,\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0my\u001b[0m \u001b[0mw\u001b[0mo\u001b[0mu\u001b[0ml\u001b[0md\u001b[0m \u001b[0m \u001b[0mB\u001b[0mo\u001b[0my\u001b[0mc\u001b[0mo\u001b[0mt\u001b[0mt\u001b[0m \u001b[0mt\u001b[0mh\u001b[0mi\u001b[0ms\u001b[0m \u001b[0m \u001b[0mi\u001b[0mn\u001b[0me\u001b[0mp\u001b[0mt\u001b[0m \u001b[0m \u001b[0ma\u001b[0mi\u001b[0mr\u001b[0ml\u001b[0mi\u001b[0mn\u001b[0me\u001b[0m,\u001b[0m \u001b[0m \u001b[0mb\u001b[0mu\u001b[0mt\u001b[0m \u001b[0m \u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0my\u001b[0m \u001b[0m \u001b[0ma\u001b[0mr\u001b[0me\u001b[0m \u001b[0m \u001b[0mn\u001b[0mo\u001b[0mt\u001b[0m \u001b[0m \u001b[0ms\u001b[0mm\u001b[0ma\u001b[0mr\u001b[0mt\u001b[0m,\u001b[0m \u001b[0m \u001b[0ms\u001b[0mo\u001b[0m \u001b[0m \u001b[0m \u001b[0mr\u001b[0mo\u001b[0mg\u001b[0mu\u001b[0me\u001b[0m \u001b[0m \u001b[0mb\u001b[0mu\u001b[0ms\u001b[0mi\u001b[0mn\u001b[0me\u001b[0ms\u001b[0ms\u001b[0me\u001b[0ms\u001b[0m \u001b[0m \u001b[0ml\u001b[0mi\u001b[0mk\u001b[0me\u001b[0m \u001b[0m \u001b[0mt\u001b[0mh\u001b[0mi\u001b[0ms\u001b[0m \u001b[0m \u001b[0mo\u001b[0mn\u001b[0me\u001b[0m,\u001b[0m \u001b[0m \u001b[0m \u001b[0ms\u001b[0mt\u001b[0mi\u001b[0ml\u001b[0ml\u001b[0m \u001b[0mt\u001b[0mh\u001b[0mr\u001b[0mi\u001b[0mv\u001b[0me\u001b[0m \u001b[0m \u001b[0m \u001b[0mt\u001b[0ma\u001b[0mk\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0m\u001b[41mi\u001b[0m\u001b[41md\u001b[0m\u001b[41mi\u001b[0m\u001b[41mo\u001b[0m\u001b[41mt\u001b[0m\u001b[41ms\u001b[0m \u001b[0m \u001b[0mf\u001b[0mo\u001b[0mr\u001b[0m \u001b[0m \u001b[0ma\u001b[0m \u001b[0mr\u001b[0mi\u001b[0md\u001b[0me\u001b[0m.\u001b[0m.\u001b[0m.\u001b[0m\n",
+      "\n",
+      "\n",
+      "3\u001b[1m\u001b[36m Pred: \u001b[0mT\u001b[0mr\u001b[0mu\u001b[0mm\u001b[0mp\u001b[0m \u001b[0mC\u001b[0ml\u001b[0ma\u001b[0mi\u001b[0mm\u001b[0me\u001b[0md\u001b[0m \u001b[0mt\u001b[0mh\u001b[0ma\u001b[0mt\u001b[0m \u001b[0mR\u001b[0mu\u001b[0ms\u001b[0ms\u001b[0mi\u001b[0ma\u001b[0m \u001b[0mw\u001b[0mi\u001b[0ml\u001b[0ml\u001b[0m \u001b[0mn\u001b[0me\u001b[0mv\u001b[0me\u001b[0mr\u001b[0m \u001b[0mi\u001b[0mn\u001b[0mv\u001b[0ma\u001b[0md\u001b[0me\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0mU\u001b[0mk\u001b[0mr\u001b[0ma\u001b[0mi\u001b[0mn\u001b[0me\u001b[0m,\u001b[0m \u001b[0mw\u001b[0mh\u001b[0me\u001b[0mn\u001b[0m \u001b[0mR\u001b[0mu\u001b[0ms\u001b[0ms\u001b[0mi\u001b[0ma\u001b[0m \u001b[0ma\u001b[0ml\u001b[0mr\u001b[0me\u001b[0ma\u001b[0md\u001b[0my\u001b[0m \u001b[0mh\u001b[0ma\u001b[0ms\u001b[0m \u001b[0m-\u001b[0m \u001b[0mh\u001b[0mo\u001b[0mw\u001b[0m \u001b[0m\u001b[41ms\u001b[0m\u001b[41mt\u001b[0m\u001b[41mu\u001b[0m\u001b[41mp\u001b[0m\u001b[41mi\u001b[0m\u001b[41md\u001b[0m \u001b[0mc\u001b[0ma\u001b[0mn\u001b[0m \u001b[0mp\u001b[0me\u001b[0mo\u001b[0mp\u001b[0ml\u001b[0me\u001b[0m \u001b[0mb\u001b[0me\u001b[0m?\u001b[0m\n",
+      "3\u001b[1m\u001b[33m Gold: \u001b[0mT\u001b[0mr\u001b[0mu\u001b[0mm\u001b[0mp\u001b[0m \u001b[0mC\u001b[0ml\u001b[0ma\u001b[0mi\u001b[0mm\u001b[0me\u001b[0md\u001b[0m \u001b[0mt\u001b[0mh\u001b[0ma\u001b[0mt\u001b[0m \u001b[0mR\u001b[0mu\u001b[0ms\u001b[0ms\u001b[0mi\u001b[0ma\u001b[0m \u001b[0mw\u001b[0mi\u001b[0ml\u001b[0ml\u001b[0m \u001b[0mn\u001b[0me\u001b[0mv\u001b[0me\u001b[0mr\u001b[0m \u001b[0mi\u001b[0mn\u001b[0mv\u001b[0ma\u001b[0md\u001b[0me\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0mU\u001b[0mk\u001b[0mr\u001b[0ma\u001b[0mi\u001b[0mn\u001b[0me\u001b[0m,\u001b[0m \u001b[0mw\u001b[0mh\u001b[0me\u001b[0mn\u001b[0m \u001b[0mR\u001b[0mu\u001b[0ms\u001b[0ms\u001b[0mi\u001b[0ma\u001b[0m \u001b[0ma\u001b[0ml\u001b[0mr\u001b[0me\u001b[0ma\u001b[0md\u001b[0my\u001b[0m \u001b[0mh\u001b[0ma\u001b[0ms\u001b[0m \u001b[0m-\u001b[0m \u001b[0mh\u001b[0mo\u001b[0mw\u001b[0m \u001b[0m\u001b[41ms\u001b[0m\u001b[41mt\u001b[0m\u001b[41mu\u001b[0m\u001b[41mp\u001b[0m\u001b[41mi\u001b[0m\u001b[41md\u001b[0m \u001b[0mc\u001b[0ma\u001b[0mn\u001b[0m \u001b[0mp\u001b[0me\u001b[0mo\u001b[0mp\u001b[0ml\u001b[0me\u001b[0m \u001b[0mb\u001b[0me\u001b[0m?\u001b[0m\n",
+      "\n",
+      "\n",
+      "4\u001b[1m\u001b[36m Pred: \u001b[0mA\u001b[0ms\u001b[0m \u001b[0ml\u001b[0mo\u001b[0mn\u001b[0mg\u001b[0m \u001b[0ma\u001b[0ms\u001b[0m \u001b[0my\u001b[0mo\u001b[0mu\u001b[0mr\u001b[0m \u001b[0mw\u001b[0mi\u001b[0ml\u001b[0ml\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m \u001b[0mt\u001b[0mo\u001b[0m \u001b[0mp\u001b[0ma\u001b[0my\u001b[0m \u001b[0ma\u001b[0m \u001b[0ml\u001b[0mo\u001b[0mt\u001b[0m \u001b[0mm\u001b[0mo\u001b[0mr\u001b[0me\u001b[0m \u001b[0mf\u001b[0mo\u001b[0mr\u001b[0m \u001b[0mp\u001b[0mr\u001b[0mo\u001b[0md\u001b[0mu\u001b[0mc\u001b[0mt\u001b[0ms\u001b[0m \u001b[0my\u001b[0mo\u001b[0mu\u001b[0m \u001b[0mb\u001b[0mu\u001b[0my\u001b[0m,\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0mn\u001b[0m \u001b[0mf\u001b[0mi\u001b[0mn\u001b[0me\u001b[0m.\u001b[0m\n",
+      "\u001b[0mB\u001b[0mu\u001b[0mt\u001b[0m \u001b[0my\u001b[0mo\u001b[0mu\u001b[0m \u001b[0mb\u001b[0me\u001b[0mt\u001b[0mt\u001b[0me\u001b[0mr\u001b[0m \u001b[0mn\u001b[0mo\u001b[0mt\u001b[0m \u001b[0mb\u001b[0me\u001b[0m \u001b[0mg\u001b[0mo\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m \u001b[0mt\u001b[0mo\u001b[0m \u001b[0mC\u001b[0mo\u001b[0ms\u001b[0mt\u001b[0mc\u001b[0mo\u001b[0m \u001b[0ma\u001b[0mn\u001b[0md\u001b[0m \u001b[0mW\u001b[0ma\u001b[0ml\u001b[0mm\u001b[0ma\u001b[0mr\u001b[0mt\u001b[0m \u001b[0mt\u001b[0mo\u001b[0m \u001b[0mb\u001b[0mu\u001b[0my\u001b[0m \u001b[0ms\u001b[0mt\u001b[0mu\u001b[0mf\u001b[0mf\u001b[0m \u001b[0mb\u001b[0me\u001b[0mc\u001b[0ma\u001b[0mu\u001b[0ms\u001b[0me\u001b[0m \u001b[0mi\u001b[0mt\u001b[0m'\u001b[0ms\u001b[0m \u001b[0mc\u001b[0mh\u001b[0me\u001b[0ma\u001b[0mp\u001b[0me\u001b[0mr\u001b[0m.\u001b[0m\n",
+      "\u001b[0mI\u001b[0mf\u001b[0m \u001b[0ms\u001b[0mo\u001b[0m,\u001b[0m \u001b[0mw\u001b[0me\u001b[0m \u001b[0mg\u001b[0me\u001b[0mt\u001b[0m \u001b[0mt\u001b[0mo\u001b[0m \u001b[0mc\u001b[0ma\u001b[0ml\u001b[0ml\u001b[0m \u001b[0my\u001b[0mo\u001b[0mu\u001b[0m \u001b[0ma\u001b[0m \u001b[0m\u001b[41mh\u001b[0m\u001b[41my\u001b[0m\u001b[41mp\u001b[0m\u001b[41mo\u001b[0m\u001b[41mc\u001b[0m\u001b[41mr\u001b[0m\u001b[41mi\u001b[0m\u001b[41mt\u001b[0m\u001b[41mi\u001b[0m\u001b[41mc\u001b[0m\u001b[41ma\u001b[0m\u001b[41ml\u001b[0m \u001b[0mw\u001b[0ma\u001b[0mn\u001b[0mk\u001b[0me\u001b[0mr\u001b[0m.\u001b[0m\n",
+      "4\u001b[1m\u001b[33m Gold: \u001b[0mA\u001b[0ms\u001b[0m \u001b[0ml\u001b[0mo\u001b[0mn\u001b[0mg\u001b[0m \u001b[0ma\u001b[0ms\u001b[0m \u001b[0my\u001b[0mo\u001b[0mu\u001b[0mr\u001b[0m \u001b[0mw\u001b[0mi\u001b[0ml\u001b[0ml\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m \u001b[0mt\u001b[0mo\u001b[0m \u001b[0mp\u001b[0ma\u001b[0my\u001b[0m \u001b[0ma\u001b[0m \u001b[0ml\u001b[0mo\u001b[0mt\u001b[0m \u001b[0mm\u001b[0mo\u001b[0mr\u001b[0me\u001b[0m \u001b[0mf\u001b[0mo\u001b[0mr\u001b[0m \u001b[0mp\u001b[0mr\u001b[0mo\u001b[0md\u001b[0mu\u001b[0mc\u001b[0mt\u001b[0ms\u001b[0m \u001b[0my\u001b[0mo\u001b[0mu\u001b[0m \u001b[0mb\u001b[0mu\u001b[0my\u001b[0m,\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0mn\u001b[0m \u001b[0mf\u001b[0mi\u001b[0mn\u001b[0me\u001b[0m.\u001b[0m\n",
+      "\u001b[0mB\u001b[0mu\u001b[0mt\u001b[0m \u001b[0my\u001b[0mo\u001b[0mu\u001b[0m \u001b[0mb\u001b[0me\u001b[0mt\u001b[0mt\u001b[0me\u001b[0mr\u001b[0m \u001b[0mn\u001b[0mo\u001b[0mt\u001b[0m \u001b[0mb\u001b[0me\u001b[0m \u001b[0mg\u001b[0mo\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m \u001b[0mt\u001b[0mo\u001b[0m \u001b[0mC\u001b[0mo\u001b[0ms\u001b[0mt\u001b[0mc\u001b[0mo\u001b[0m \u001b[0ma\u001b[0mn\u001b[0md\u001b[0m \u001b[0mW\u001b[0ma\u001b[0ml\u001b[0mm\u001b[0ma\u001b[0mr\u001b[0mt\u001b[0m \u001b[0mt\u001b[0mo\u001b[0m \u001b[0mb\u001b[0mu\u001b[0my\u001b[0m \u001b[0ms\u001b[0mt\u001b[0mu\u001b[0mf\u001b[0mf\u001b[0m \u001b[0mb\u001b[0me\u001b[0mc\u001b[0ma\u001b[0mu\u001b[0ms\u001b[0me\u001b[0m \u001b[0mi\u001b[0mt\u001b[0m'\u001b[0ms\u001b[0m \u001b[0mc\u001b[0mh\u001b[0me\u001b[0ma\u001b[0mp\u001b[0me\u001b[0mr\u001b[0m.\u001b[0m\n",
+      "\u001b[0mI\u001b[0mf\u001b[0m \u001b[0ms\u001b[0mo\u001b[0m,\u001b[0m \u001b[0mw\u001b[0me\u001b[0m \u001b[0mg\u001b[0me\u001b[0mt\u001b[0m \u001b[0mt\u001b[0mo\u001b[0m \u001b[0mc\u001b[0ma\u001b[0ml\u001b[0ml\u001b[0m \u001b[0my\u001b[0mo\u001b[0mu\u001b[0m \u001b[0ma\u001b[0m \u001b[0mh\u001b[0my\u001b[0mp\u001b[0mo\u001b[0mc\u001b[0mr\u001b[0mi\u001b[0mt\u001b[0mi\u001b[0mc\u001b[0ma\u001b[0ml\u001b[0m \u001b[0mw\u001b[0ma\u001b[0mn\u001b[0mk\u001b[0me\u001b[0mr\u001b[0m.\u001b[0m\n",
+      "\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "indices_test = []\n",
+    "for i, (gold_index, text) in enumerate(zip(test['spans'],test['text'])):\n",
+    "    tagged_sentence = tagger_LSTM(text)   \n",
+    "    prediction_index = get_index_toxic_words(text.lower(), tagged_sentence)\n",
+    "    indices_test.append(prediction_index)\n",
+    "    \n",
+    "    if i < 5:\n",
+    "        print(str(i) + colored(' Pred: ', color='cyan', attrs=['bold']) + \n",
+    "              color_toxic_words(prediction_index, text))\n",
+    "        print(str(i) + colored(' Gold: ', color='yellow', attrs=['bold']) + \n",
+    "              color_toxic_words(gold_index, text) + '\\n'*2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "F1 in test: 0.648722\n"
+     ]
+    }
+   ],
+   "source": [
+    "score_test = [f1(pred, gold) for pred,gold in zip(indices_test, test['spans'])]\n",
+    "print('F1 in test: {:.6f}'.format(np.mean(score_test)))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Predicción en el dataset de *evaluation*"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "evaluation = pd.read_csv('../data/tsd_test.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0\u001b[1m\u001b[36m Pred: \u001b[0mT\u001b[0mh\u001b[0ma\u001b[0mt\u001b[0m'\u001b[0ms\u001b[0m \u001b[0mr\u001b[0mi\u001b[0mg\u001b[0mh\u001b[0mt\u001b[0m.\u001b[0m \u001b[0mT\u001b[0mh\u001b[0me\u001b[0my\u001b[0m \u001b[0ma\u001b[0mr\u001b[0me\u001b[0m \u001b[0mn\u001b[0mo\u001b[0mt\u001b[0m \u001b[0mn\u001b[0mo\u001b[0mr\u001b[0mm\u001b[0ma\u001b[0ml\u001b[0m.\u001b[0m \u001b[0mA\u001b[0mn\u001b[0md\u001b[0m \u001b[0mI\u001b[0m \u001b[0ma\u001b[0mm\u001b[0m \u001b[0ms\u001b[0mt\u001b[0ma\u001b[0mr\u001b[0mt\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m \u001b[0mf\u001b[0mr\u001b[0mo\u001b[0mm\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0mp\u001b[0mr\u001b[0me\u001b[0mm\u001b[0mi\u001b[0ms\u001b[0me\u001b[0m \u001b[0mt\u001b[0mh\u001b[0ma\u001b[0mt\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0my\u001b[0m \u001b[0ma\u001b[0mr\u001b[0me\u001b[0m \u001b[0mA\u001b[0mB\u001b[0mN\u001b[0mO\u001b[0mR\u001b[0mM\u001b[0mA\u001b[0mL\u001b[0m.\u001b[0m \u001b[0mP\u001b[0mr\u001b[0mo\u001b[0mc\u001b[0me\u001b[0me\u001b[0md\u001b[0m \u001b[0mw\u001b[0mt\u001b[0mh\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0mt\u001b[0my\u001b[0mp\u001b[0mi\u001b[0mc\u001b[0ma\u001b[0ml\u001b[0m \u001b[0m\u001b[41mr\u001b[0m\u001b[41ma\u001b[0m\u001b[41mc\u001b[0m\u001b[41mi\u001b[0m\u001b[41ms\u001b[0m\u001b[41mt\u001b[0m\u001b[41m,\u001b[0m\u001b[41m \u001b[0m\u001b[41mb\u001b[0m\u001b[41mi\u001b[0m\u001b[41mg\u001b[0m\u001b[41mo\u001b[0m\u001b[41mt\u001b[0m\u001b[41m,\u001b[0m\u001b[41m \u001b[0m\u001b[41ms\u001b[0m\u001b[41me\u001b[0m\u001b[41mx\u001b[0m\u001b[41mi\u001b[0m\u001b[41ms\u001b[0m\u001b[41mt\u001b[0m\u001b[41m \u001b[0m\u001b[41mr\u001b[0m\u001b[41mu\u001b[0m\u001b[41mb\u001b[0m\u001b[41mb\u001b[0m\u001b[41mi\u001b[0m\u001b[41ms\u001b[0m\u001b[41mh\u001b[0m.\u001b[0m\n",
+      "\u001b[0mT\u001b[0mh\u001b[0ma\u001b[0mn\u001b[0mk\u001b[0ms\u001b[0m!\u001b[0m\n",
+      "\n",
+      "1\u001b[1m\u001b[36m Pred: \u001b[0m\"\u001b[0mW\u001b[0ma\u001b[0mt\u001b[0mc\u001b[0mh\u001b[0m \u001b[0mp\u001b[0me\u001b[0mo\u001b[0mp\u001b[0ml\u001b[0me\u001b[0m \u001b[0md\u001b[0mi\u001b[0me\u001b[0m \u001b[0mf\u001b[0mr\u001b[0mo\u001b[0mm\u001b[0m \u001b[0mt\u001b[0ma\u001b[0mk\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m \u001b[0ma\u001b[0mw\u001b[0ma\u001b[0my\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0mi\u001b[0mr\u001b[0m \u001b[0mh\u001b[0me\u001b[0ma\u001b[0ml\u001b[0mt\u001b[0mh\u001b[0mc\u001b[0ma\u001b[0mr\u001b[0me\u001b[0m\"\u001b[0m\n",
+      "\u001b[0m\n",
+      "\u001b[0mD\u001b[0mI\u001b[0mN\u001b[0mG\u001b[0m \u001b[0mD\u001b[0mI\u001b[0mN\u001b[0mG\u001b[0m \u001b[0mD\u001b[0mI\u001b[0mN\u001b[0mG\u001b[0m!\u001b[0m \u001b[0m \u001b[0mW\u001b[0mi\u001b[0mn\u001b[0mn\u001b[0me\u001b[0mr\u001b[0m \u001b[0mo\u001b[0mf\u001b[0m \u001b[0m\u001b[41ms\u001b[0m\u001b[41mt\u001b[0m\u001b[41mu\u001b[0m\u001b[41mp\u001b[0m\u001b[41mi\u001b[0m\u001b[41md\u001b[0m \u001b[0mp\u001b[0mo\u001b[0ms\u001b[0mt\u001b[0m \u001b[0mo\u001b[0mf\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0md\u001b[0ma\u001b[0my\u001b[0m \u001b[0ma\u001b[0mw\u001b[0ma\u001b[0mr\u001b[0md\u001b[0m!\u001b[0m\n",
+      "\n",
+      "2\u001b[1m\u001b[36m Pred: \u001b[0mt\u001b[0me\u001b[0mn\u001b[0ms\u001b[0m \u001b[0my\u001b[0me\u001b[0ma\u001b[0mr\u001b[0ms\u001b[0m \u001b[0ma\u001b[0mg\u001b[0mo\u001b[0m \u001b[0mi\u001b[0m \u001b[0mc\u001b[0mo\u001b[0mn\u001b[0mt\u001b[0ma\u001b[0mc\u001b[0mt\u001b[0me\u001b[0md\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0mP\u001b[0mD\u001b[0mR\u001b[0m \u001b[0ma\u001b[0mn\u001b[0md\u001b[0m \u001b[0ms\u001b[0mu\u001b[0mg\u001b[0mg\u001b[0me\u001b[0ms\u001b[0mt\u001b[0me\u001b[0md\u001b[0m \u001b[0mt\u001b[0mh\u001b[0ma\u001b[0mt\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0mt\u001b[0mi\u001b[0mm\u001b[0me\u001b[0m \u001b[0mm\u001b[0mi\u001b[0mg\u001b[0mh\u001b[0mt\u001b[0m \u001b[0mb\u001b[0me\u001b[0m \u001b[0mg\u001b[0mo\u001b[0mo\u001b[0md\u001b[0m \u001b[0mt\u001b[0mo\u001b[0m \u001b[0mw\u001b[0mo\u001b[0mr\u001b[0mk\u001b[0m \u001b[0mw\u001b[0mi\u001b[0mt\u001b[0mh\u001b[0m \u001b[0ma\u001b[0ml\u001b[0ma\u001b[0ms\u001b[0mk\u001b[0ma\u001b[0m \u001b[0mo\u001b[0mn\u001b[0m \u001b[0mb\u001b[0mu\u001b[0mi\u001b[0ml\u001b[0md\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m \u001b[0ma\u001b[0m \u001b[0mg\u001b[0ma\u001b[0ms\u001b[0m \u001b[0ml\u001b[0mi\u001b[0mn\u001b[0me\u001b[0m.\u001b[0m.\u001b[0m \u001b[0ma\u001b[0ml\u001b[0ma\u001b[0ms\u001b[0mk\u001b[0ma\u001b[0m \u001b[0mr\u001b[0me\u001b[0mj\u001b[0me\u001b[0mc\u001b[0mt\u001b[0me\u001b[0md\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0mm\u001b[0m \u001b[0mw\u001b[0mi\u001b[0mt\u001b[0mh\u001b[0mo\u001b[0mu\u001b[0mt\u001b[0m \u001b[0me\u001b[0mv\u001b[0me\u001b[0mn\u001b[0m \u001b[0mc\u001b[0mo\u001b[0mn\u001b[0ms\u001b[0mi\u001b[0md\u001b[0me\u001b[0mr\u001b[0ma\u001b[0mt\u001b[0mi\u001b[0mo\u001b[0mn\u001b[0m \u001b[0md\u001b[0me\u001b[0ms\u001b[0mp\u001b[0mi\u001b[0mt\u001b[0me\u001b[0m \u001b[0mc\u001b[0mh\u001b[0mi\u001b[0mn\u001b[0ma\u001b[0m \u001b[0mb\u001b[0me\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m \u001b[0mf\u001b[0ml\u001b[0mu\u001b[0ms\u001b[0mh\u001b[0m \u001b[0mw\u001b[0mi\u001b[0mt\u001b[0mh\u001b[0m \u001b[0mc\u001b[0ma\u001b[0ms\u001b[0mh\u001b[0m \u001b[0ma\u001b[0mn\u001b[0md\u001b[0m \u001b[0mh\u001b[0mu\u001b[0mn\u001b[0mg\u001b[0mr\u001b[0my\u001b[0m \u001b[0mf\u001b[0mo\u001b[0mr\u001b[0m \u001b[0mg\u001b[0ma\u001b[0ms\u001b[0m.\u001b[0m.\u001b[0m \u001b[0ma\u001b[0mn\u001b[0md\u001b[0m \u001b[0ms\u001b[0me\u001b[0mt\u001b[0m \u001b[0mu\u001b[0mp\u001b[0m \u001b[0ma\u001b[0mn\u001b[0mo\u001b[0mt\u001b[0mh\u001b[0me\u001b[0mr\u001b[0m \u001b[0mi\u001b[0mn\u001b[0mf\u001b[0ma\u001b[0mm\u001b[0mo\u001b[0mu\u001b[0ms\u001b[0m \u001b[0mb\u001b[0mo\u001b[0mo\u001b[0mn\u001b[0md\u001b[0mo\u001b[0mg\u001b[0mg\u001b[0ml\u001b[0me\u001b[0m.\u001b[0m.\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0mt\u001b[0mr\u001b[0ma\u001b[0mn\u001b[0ms\u001b[0mc\u001b[0ma\u001b[0mn\u001b[0ma\u001b[0md\u001b[0ma\u001b[0m-\u001b[0me\u001b[0mx\u001b[0mx\u001b[0mo\u001b[0mn\u001b[0m \u001b[0mr\u001b[0mi\u001b[0mp\u001b[0m \u001b[0mo\u001b[0mf\u001b[0mf\u001b[0m \u001b[0mt\u001b[0mh\u001b[0ma\u001b[0mt\u001b[0m \u001b[0mw\u001b[0me\u001b[0m \u001b[0ma\u001b[0mr\u001b[0me\u001b[0m \u001b[0ms\u001b[0mt\u001b[0mi\u001b[0ml\u001b[0ml\u001b[0m \u001b[0mp\u001b[0ma\u001b[0my\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m \u001b[0mf\u001b[0mo\u001b[0mr\u001b[0m \u001b[0ma\u001b[0mn\u001b[0md\u001b[0m \u001b[0mh\u001b[0ma\u001b[0mv\u001b[0me\u001b[0m \u001b[0my\u001b[0me\u001b[0mt\u001b[0m \u001b[0mt\u001b[0mo\u001b[0m \u001b[0mr\u001b[0me\u001b[0mc\u001b[0me\u001b[0mi\u001b[0mv\u001b[0me\u001b[0m \u001b[0ma\u001b[0mn\u001b[0my\u001b[0mt\u001b[0mh\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m \u001b[0mo\u001b[0mf\u001b[0m \u001b[0mv\u001b[0ma\u001b[0ml\u001b[0mu\u001b[0me\u001b[0m.\u001b[0m.\u001b[0m \u001b[0mh\u001b[0mu\u001b[0mn\u001b[0md\u001b[0mr\u001b[0me\u001b[0md\u001b[0ms\u001b[0m \u001b[0mo\u001b[0mf\u001b[0m \u001b[0mm\u001b[0mi\u001b[0ml\u001b[0ml\u001b[0mi\u001b[0mo\u001b[0mn\u001b[0ms\u001b[0m \u001b[0mo\u001b[0mf\u001b[0m \u001b[0md\u001b[0mo\u001b[0ml\u001b[0ml\u001b[0ma\u001b[0mr\u001b[0ms\u001b[0m \u001b[0mo\u001b[0mn\u001b[0m \u001b[0ms\u001b[0mt\u001b[0mu\u001b[0md\u001b[0mi\u001b[0me\u001b[0ms\u001b[0m.\u001b[0m.\u001b[0m \u001b[0ma\u001b[0mn\u001b[0md\u001b[0m \u001b[0mb\u001b[0mu\u001b[0my\u001b[0mo\u001b[0mu\u001b[0mt\u001b[0ms\u001b[0m.\u001b[0m.\u001b[0m \u001b[0mi\u001b[0m \u001b[0mh\u001b[0mo\u001b[0mp\u001b[0me\u001b[0m \u001b[0mc\u001b[0mh\u001b[0mi\u001b[0mn\u001b[0ma\u001b[0m \u001b[0ms\u001b[0ma\u001b[0my\u001b[0ms\u001b[0m \u001b[0mf\u001b[0m \u001b[0my\u001b[0mo\u001b[0mu\u001b[0m \u001b[0ma\u001b[0ml\u001b[0ma\u001b[0ms\u001b[0mk\u001b[0ma\u001b[0m.\u001b[0m.\u001b[0m \u001b[0my\u001b[0mo\u001b[0mu\u001b[0m \u001b[0ma\u001b[0mr\u001b[0me\u001b[0m \u001b[0mn\u001b[0mo\u001b[0mt\u001b[0mh\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m \u001b[0mb\u001b[0mu\u001b[0mt\u001b[0m \u001b[0m\u001b[41mi\u001b[0m\u001b[41mg\u001b[0m\u001b[41mn\u001b[0m\u001b[41mo\u001b[0m\u001b[41mr\u001b[0m\u001b[41ma\u001b[0m\u001b[41mn\u001b[0m\u001b[41mt\u001b[0m \u001b[0mp\u001b[0me\u001b[0mo\u001b[0mp\u001b[0ml\u001b[0me\u001b[0m.\u001b[0m.\u001b[0m\n",
+      "\n",
+      "3\u001b[1m\u001b[36m Pred: \u001b[0mT\u001b[0mh\u001b[0me\u001b[0m \u001b[0mp\u001b[0ma\u001b[0mr\u001b[0ma\u001b[0ml\u001b[0ml\u001b[0me\u001b[0ml\u001b[0ms\u001b[0m \u001b[0mb\u001b[0me\u001b[0mt\u001b[0mw\u001b[0me\u001b[0me\u001b[0mn\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0mA\u001b[0mN\u001b[0mC\u001b[0m \u001b[0ma\u001b[0mn\u001b[0md\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0mS\u001b[0mi\u001b[0mc\u001b[0mi\u001b[0ml\u001b[0mi\u001b[0ma\u001b[0mn\u001b[0m \u001b[0mM\u001b[0ma\u001b[0mf\u001b[0mi\u001b[0ma\u001b[0m \u001b[0ma\u001b[0mr\u001b[0me\u001b[0m \u001b[0mg\u001b[0ml\u001b[0ma\u001b[0mr\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m.\u001b[0m \u001b[0mT\u001b[0mh\u001b[0me\u001b[0m \u001b[0mA\u001b[0mN\u001b[0mC\u001b[0m \u001b[0mh\u001b[0ma\u001b[0ms\u001b[0m \u001b[0ma\u001b[0ml\u001b[0mw\u001b[0ma\u001b[0my\u001b[0ms\u001b[0m \u001b[0mb\u001b[0me\u001b[0me\u001b[0mn\u001b[0m \u001b[0mr\u001b[0mu\u001b[0mn\u001b[0m \u001b[0mb\u001b[0my\u001b[0m \u001b[0ma\u001b[0m \u001b[0mf\u001b[0me\u001b[0mw\u001b[0m \u001b[0m\"\u001b[0mf\u001b[0ma\u001b[0mm\u001b[0mi\u001b[0ml\u001b[0mi\u001b[0me\u001b[0ms\u001b[0m\"\u001b[0m \u001b[0mw\u001b[0mh\u001b[0mo\u001b[0m \u001b[0mt\u001b[0mr\u001b[0me\u001b[0ma\u001b[0mt\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0ms\u001b[0mt\u001b[0ma\u001b[0mt\u001b[0me\u001b[0m \u001b[0ma\u001b[0ms\u001b[0m \u001b[0m'\u001b[0mt\u001b[0mu\u001b[0mr\u001b[0mf\u001b[0m'\u001b[0m \u001b[0m;\u001b[0m \u001b[0ma\u001b[0ms\u001b[0m \u001b[0mj\u001b[0mu\u001b[0ms\u001b[0mt\u001b[0m \u001b[0mo\u001b[0mn\u001b[0me\u001b[0m \u001b[0mb\u001b[0mi\u001b[0mg\u001b[0m \u001b[0mp\u001b[0mi\u001b[0mg\u001b[0mg\u001b[0my\u001b[0m \u001b[0mb\u001b[0ma\u001b[0mn\u001b[0mk\u001b[0m \u001b[0mf\u001b[0mo\u001b[0mr\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0mi\u001b[0mr\u001b[0m \u001b[0ms\u001b[0me\u001b[0ml\u001b[0mf\u001b[0m-\u001b[0me\u001b[0mn\u001b[0mr\u001b[0mi\u001b[0mc\u001b[0mh\u001b[0mm\u001b[0me\u001b[0mn\u001b[0mt\u001b[0m.\u001b[0m \u001b[0mT\u001b[0mh\u001b[0me\u001b[0m \u001b[0mg\u001b[0mo\u001b[0mv\u001b[0me\u001b[0mr\u001b[0mn\u001b[0mm\u001b[0me\u001b[0mn\u001b[0mt\u001b[0m \u001b[0mb\u001b[0ma\u001b[0ms\u001b[0mi\u001b[0mc\u001b[0ma\u001b[0ml\u001b[0ml\u001b[0my\u001b[0m \u001b[0mb\u001b[0me\u001b[0ml\u001b[0mi\u001b[0me\u001b[0mv\u001b[0me\u001b[0ms\u001b[0m \u001b[0mw\u001b[0me\u001b[0m \u001b[0ma\u001b[0ml\u001b[0ml\u001b[0m \u001b[0mj\u001b[0mu\u001b[0ms\u001b[0mt\u001b[0m \u001b[0mw\u001b[0mo\u001b[0mr\u001b[0mk\u001b[0m \u001b[0mf\u001b[0mo\u001b[0mr\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0mm\u001b[0m.\u001b[0m \u001b[0mT\u001b[0mh\u001b[0me\u001b[0my\u001b[0m \u001b[0ma\u001b[0mr\u001b[0me\u001b[0mn\u001b[0m'\u001b[0mt\u001b[0m \u001b[0ma\u001b[0m \u001b[0md\u001b[0me\u001b[0mm\u001b[0mo\u001b[0mc\u001b[0mr\u001b[0ma\u001b[0mt\u001b[0mi\u001b[0mc\u001b[0m \u001b[0mg\u001b[0mo\u001b[0mv\u001b[0me\u001b[0mr\u001b[0mn\u001b[0mm\u001b[0me\u001b[0mn\u001b[0mt\u001b[0m \u001b[0ma\u001b[0mt\u001b[0m \u001b[0ma\u001b[0ml\u001b[0ml\u001b[0m,\u001b[0m \u001b[0mb\u001b[0mu\u001b[0mt\u001b[0m \u001b[0mu\u001b[0ms\u001b[0me\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0ma\u001b[0mp\u001b[0mp\u001b[0me\u001b[0ma\u001b[0mr\u001b[0ma\u001b[0mn\u001b[0mc\u001b[0me\u001b[0ms\u001b[0m \u001b[0mo\u001b[0mf\u001b[0m \u001b[0md\u001b[0me\u001b[0mm\u001b[0mo\u001b[0mc\u001b[0mr\u001b[0ma\u001b[0mc\u001b[0my\u001b[0m \u001b[0mt\u001b[0mo\u001b[0m \u001b[0mg\u001b[0mi\u001b[0mv\u001b[0me\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0mi\u001b[0mr\u001b[0m \u001b[0me\u001b[0mn\u001b[0mt\u001b[0mi\u001b[0mt\u001b[0ml\u001b[0me\u001b[0mm\u001b[0me\u001b[0mn\u001b[0mt\u001b[0m \u001b[0mp\u001b[0mr\u001b[0ma\u001b[0mc\u001b[0mt\u001b[0mi\u001b[0mc\u001b[0me\u001b[0ms\u001b[0m \u001b[0ma\u001b[0m \u001b[0mm\u001b[0ma\u001b[0ms\u001b[0mk\u001b[0m \u001b[0mo\u001b[0mf\u001b[0m \u001b[0ml\u001b[0me\u001b[0mg\u001b[0mi\u001b[0mt\u001b[0mi\u001b[0mm\u001b[0ma\u001b[0mc\u001b[0my\u001b[0m.\u001b[0m \u001b[0mT\u001b[0mh\u001b[0me\u001b[0m \u001b[0mp\u001b[0mo\u001b[0mo\u001b[0mr\u001b[0m \u001b[0ma\u001b[0mn\u001b[0md\u001b[0m \u001b[0m\u001b[41mi\u001b[0m\u001b[41mg\u001b[0m\u001b[41mn\u001b[0m\u001b[41mo\u001b[0m\u001b[41mr\u001b[0m\u001b[41ma\u001b[0m\u001b[41mn\u001b[0m\u001b[41mt\u001b[0m \u001b[0mh\u001b[0ma\u001b[0mv\u001b[0me\u001b[0m \u001b[0mb\u001b[0me\u001b[0me\u001b[0mn\u001b[0m \u001b[0mf\u001b[0mo\u001b[0mo\u001b[0ml\u001b[0me\u001b[0md\u001b[0m \u001b[0mf\u001b[0mo\u001b[0mr\u001b[0m \u001b[0ms\u001b[0mo\u001b[0m \u001b[0ml\u001b[0mo\u001b[0mn\u001b[0mg\u001b[0m,\u001b[0m \u001b[0mb\u001b[0mu\u001b[0mt\u001b[0m \u001b[0mp\u001b[0me\u001b[0mo\u001b[0mp\u001b[0ml\u001b[0me\u001b[0m \u001b[0ma\u001b[0mr\u001b[0me\u001b[0m \u001b[0ms\u001b[0ml\u001b[0mo\u001b[0mw\u001b[0ml\u001b[0my\u001b[0m \u001b[0mc\u001b[0mo\u001b[0mm\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m \u001b[0mt\u001b[0mo\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0mA\u001b[0mN\u001b[0mC\u001b[0m \u001b[0mf\u001b[0mo\u001b[0mr\u001b[0m \u001b[0mw\u001b[0mh\u001b[0ma\u001b[0mt\u001b[0m \u001b[0mi\u001b[0mt\u001b[0m \u001b[0mi\u001b[0ms\u001b[0m:\u001b[0m \u001b[0ma\u001b[0m \u001b[0ms\u001b[0me\u001b[0ml\u001b[0mf\u001b[0m-\u001b[0ms\u001b[0me\u001b[0mr\u001b[0mv\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m \u001b[0mM\u001b[0ma\u001b[0mf\u001b[0mi\u001b[0ma\u001b[0m!\u001b[0m\n",
+      "\n",
+      "4\u001b[1m\u001b[36m Pred: \u001b[0mI\u001b[0mn\u001b[0mt\u001b[0me\u001b[0ml\u001b[0m \u001b[0mC\u001b[0mo\u001b[0mm\u001b[0mm\u001b[0mu\u001b[0mn\u001b[0mi\u001b[0mt\u001b[0my\u001b[0m:\u001b[0m \u001b[0m‘\u001b[0mH\u001b[0mo\u001b[0mw\u001b[0m \u001b[0mc\u001b[0ma\u001b[0mn\u001b[0m \u001b[0mw\u001b[0me\u001b[0m \u001b[0mw\u001b[0mo\u001b[0mr\u001b[0mk\u001b[0m \u001b[0mf\u001b[0mo\u001b[0mr\u001b[0m \u001b[0ma\u001b[0m \u001b[0mP\u001b[0mr\u001b[0me\u001b[0ms\u001b[0mi\u001b[0md\u001b[0me\u001b[0mn\u001b[0mt\u001b[0m \u001b[0mw\u001b[0mh\u001b[0mo\u001b[0m \u001b[0mU\u001b[0mn\u001b[0md\u001b[0me\u001b[0mr\u001b[0mm\u001b[0mi\u001b[0mn\u001b[0me\u001b[0ms\u001b[0m \u001b[0mo\u001b[0mu\u001b[0mr\u001b[0m \u001b[0mw\u001b[0mo\u001b[0mr\u001b[0mk\u001b[0m?\u001b[0m’\u001b[0m\n",
+      "\u001b[0m\n",
+      "\u001b[0mD\u001b[0ma\u001b[0my\u001b[0ms\u001b[0m \u001b[0mb\u001b[0me\u001b[0mf\u001b[0mo\u001b[0mr\u001b[0me\u001b[0m \u001b[0mt\u001b[0ma\u001b[0mk\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m \u001b[0mo\u001b[0mf\u001b[0mf\u001b[0mi\u001b[0mc\u001b[0me\u001b[0m,\u001b[0m \u001b[0mh\u001b[0me\u001b[0m \u001b[0mc\u001b[0mo\u001b[0mm\u001b[0mp\u001b[0ma\u001b[0mr\u001b[0me\u001b[0md\u001b[0m \u001b[0mA\u001b[0mm\u001b[0me\u001b[0mr\u001b[0mi\u001b[0mc\u001b[0ma\u001b[0m’\u001b[0ms\u001b[0m \u001b[0ms\u001b[0mp\u001b[0mi\u001b[0me\u001b[0ms\u001b[0m \u001b[0mt\u001b[0mo\u001b[0m \u001b[0mA\u001b[0md\u001b[0mo\u001b[0ml\u001b[0mf\u001b[0m \u001b[0mH\u001b[0mi\u001b[0mt\u001b[0ml\u001b[0me\u001b[0mr\u001b[0m’\u001b[0ms\u001b[0m \u001b[0m\u001b[41mG\u001b[0m\u001b[41me\u001b[0m\u001b[41ms\u001b[0m\u001b[41mt\u001b[0m\u001b[41ma\u001b[0m\u001b[41mp\u001b[0m\u001b[41mo\u001b[0m.\u001b[0m\n",
+      "\u001b[0m\n",
+      "\u001b[0mA\u001b[0mm\u001b[0me\u001b[0mr\u001b[0mi\u001b[0mc\u001b[0ma\u001b[0mn\u001b[0m \u001b[0ms\u001b[0mp\u001b[0mi\u001b[0me\u001b[0ms\u001b[0m \u001b[0md\u001b[0mo\u001b[0m \u001b[0mn\u001b[0mo\u001b[0mt\u001b[0m \u001b[0ml\u001b[0mi\u001b[0mk\u001b[0me\u001b[0m \u001b[0mh\u001b[0ma\u001b[0mv\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0mi\u001b[0mr\u001b[0m \u001b[0mw\u001b[0mo\u001b[0mr\u001b[0mk\u001b[0m \u001b[0md\u001b[0mi\u001b[0ms\u001b[0mm\u001b[0mi\u001b[0ms\u001b[0ms\u001b[0me\u001b[0md\u001b[0m \u001b[0mb\u001b[0my\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0mp\u001b[0mr\u001b[0me\u001b[0ms\u001b[0mi\u001b[0md\u001b[0me\u001b[0mn\u001b[0mt\u001b[0m.\u001b[0m \u001b[0mN\u001b[0mo\u001b[0mr\u001b[0m \u001b[0md\u001b[0mo\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0my\u001b[0m \u001b[0ma\u001b[0mp\u001b[0mp\u001b[0mr\u001b[0me\u001b[0mc\u001b[0mi\u001b[0ma\u001b[0mt\u001b[0me\u001b[0m \u001b[0mc\u001b[0mo\u001b[0mm\u001b[0mp\u001b[0ma\u001b[0mr\u001b[0mi\u001b[0ms\u001b[0mo\u001b[0mn\u001b[0ms\u001b[0m \u001b[0mt\u001b[0mo\u001b[0m \u001b[0m\u001b[41mN\u001b[0m\u001b[41ma\u001b[0m\u001b[41mz\u001b[0m\u001b[41mi\u001b[0m \u001b[0mG\u001b[0me\u001b[0mr\u001b[0mm\u001b[0ma\u001b[0mn\u001b[0my\u001b[0m.\u001b[0m\n",
+      "\u001b[0m\n",
+      "\u001b[0mF\u001b[0mo\u001b[0mr\u001b[0mm\u001b[0me\u001b[0mr\u001b[0m \u001b[0mC\u001b[0mI\u001b[0mA\u001b[0m \u001b[0mD\u001b[0mi\u001b[0mr\u001b[0me\u001b[0mc\u001b[0mt\u001b[0mo\u001b[0mr\u001b[0m \u001b[0mJ\u001b[0mo\u001b[0mh\u001b[0mn\u001b[0m \u001b[0mB\u001b[0mr\u001b[0me\u001b[0mn\u001b[0mn\u001b[0ma\u001b[0mn\u001b[0m \u001b[0mm\u001b[0ma\u001b[0md\u001b[0me\u001b[0m \u001b[0mi\u001b[0mt\u001b[0m \u001b[0mc\u001b[0ml\u001b[0me\u001b[0ma\u001b[0mr\u001b[0m:\u001b[0m \u001b[0m“\u001b[0mT\u001b[0mh\u001b[0me\u001b[0m \u001b[0mp\u001b[0me\u001b[0mr\u001b[0ms\u001b[0mo\u001b[0mn\u001b[0m \u001b[0mw\u001b[0mh\u001b[0mo\u001b[0m \u001b[0ms\u001b[0ma\u001b[0mi\u001b[0md\u001b[0m \u001b[0mt\u001b[0mh\u001b[0ma\u001b[0mt\u001b[0m \u001b[0ms\u001b[0mh\u001b[0mo\u001b[0mu\u001b[0ml\u001b[0md\u001b[0m \u001b[0mb\u001b[0me\u001b[0m \u001b[0ma\u001b[0ms\u001b[0mh\u001b[0ma\u001b[0mm\u001b[0me\u001b[0md\u001b[0m \u001b[0mo\u001b[0mf\u001b[0m \u001b[0mh\u001b[0mi\u001b[0mm\u001b[0ms\u001b[0me\u001b[0ml\u001b[0mf\u001b[0m,\u001b[0m”\u001b[0m \u001b[0mB\u001b[0mr\u001b[0me\u001b[0mn\u001b[0mn\u001b[0ma\u001b[0mn\u001b[0m \u001b[0ms\u001b[0ma\u001b[0mi\u001b[0md\u001b[0m.\u001b[0m\n",
+      "\u001b[0m\n",
+      "\u001b[0mC\u001b[0mo\u001b[0ma\u001b[0mt\u001b[0ms\u001b[0m’\u001b[0m \u001b[0mp\u001b[0mr\u001b[0me\u001b[0md\u001b[0me\u001b[0mc\u001b[0me\u001b[0ms\u001b[0ms\u001b[0mo\u001b[0mr\u001b[0m,\u001b[0m \u001b[0mG\u001b[0me\u001b[0mn\u001b[0m.\u001b[0m \u001b[0mJ\u001b[0ma\u001b[0mm\u001b[0me\u001b[0ms\u001b[0m \u001b[0mC\u001b[0ml\u001b[0ma\u001b[0mp\u001b[0mp\u001b[0me\u001b[0mr\u001b[0m,\u001b[0m \u001b[0mc\u001b[0ma\u001b[0ml\u001b[0ml\u001b[0me\u001b[0md\u001b[0m \u001b[0mT\u001b[0mr\u001b[0mu\u001b[0mm\u001b[0mp\u001b[0m’\u001b[0ms\u001b[0m \u001b[0mc\u001b[0mo\u001b[0mm\u001b[0mm\u001b[0me\u001b[0mn\u001b[0mt\u001b[0ms\u001b[0m \u001b[0m“\u001b[0ma\u001b[0m \u001b[0mt\u001b[0me\u001b[0mr\u001b[0mr\u001b[0mi\u001b[0mb\u001b[0ml\u001b[0me\u001b[0m,\u001b[0m \u001b[0mi\u001b[0mn\u001b[0ms\u001b[0mu\u001b[0ml\u001b[0mt\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m \u001b[0ma\u001b[0mf\u001b[0mf\u001b[0mr\u001b[0mo\u001b[0mn\u001b[0mt\u001b[0m \u001b[0m…\u001b[0m \u001b[0mc\u001b[0mo\u001b[0mm\u001b[0mp\u001b[0ml\u001b[0me\u001b[0mt\u001b[0me\u001b[0ml\u001b[0my\u001b[0m \u001b[0mi\u001b[0mn\u001b[0ma\u001b[0mp\u001b[0mp\u001b[0mr\u001b[0mo\u001b[0mp\u001b[0mr\u001b[0mi\u001b[0ma\u001b[0mt\u001b[0me\u001b[0m.\u001b[0m”\u001b[0m\n",
+      "\u001b[0m\n",
+      "\u001b[0mT\u001b[0mr\u001b[0mu\u001b[0mm\u001b[0mp\u001b[0m’\u001b[0ms\u001b[0m \u001b[0mo\u001b[0mp\u001b[0mi\u001b[0mn\u001b[0mi\u001b[0mo\u001b[0mn\u001b[0ms\u001b[0m \u001b[0ma\u001b[0mr\u001b[0me\u001b[0m \u001b[0mo\u001b[0mu\u001b[0mt\u001b[0m \u001b[0mo\u001b[0mf\u001b[0m \u001b[0ms\u001b[0mt\u001b[0me\u001b[0mp\u001b[0m \u001b[0mw\u001b[0mi\u001b[0mt\u001b[0mh\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0mc\u001b[0mo\u001b[0mn\u001b[0mc\u001b[0ml\u001b[0mu\u001b[0ms\u001b[0mi\u001b[0mo\u001b[0mn\u001b[0ms\u001b[0m \u001b[0mo\u001b[0mf\u001b[0m \u001b[0mt\u001b[0mh\u001b[0me\u001b[0m \u001b[0ma\u001b[0mg\u001b[0me\u001b[0mn\u001b[0mc\u001b[0mi\u001b[0me\u001b[0ms\u001b[0m \u001b[0mh\u001b[0me\u001b[0m \u001b[0mi\u001b[0ms\u001b[0m \u001b[0ms\u001b[0mu\u001b[0mp\u001b[0mp\u001b[0mo\u001b[0ms\u001b[0me\u001b[0md\u001b[0m \u001b[0mt\u001b[0mo\u001b[0m \u001b[0mb\u001b[0me\u001b[0m \u001b[0ml\u001b[0me\u001b[0ma\u001b[0md\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m.\u001b[0m\n",
+      "\u001b[0m\n",
+      "\u001b[0mN\u001b[0mo\u001b[0mt\u001b[0m \u001b[0ms\u001b[0mu\u001b[0mr\u001b[0mp\u001b[0mr\u001b[0mi\u001b[0ms\u001b[0mi\u001b[0mn\u001b[0mg\u001b[0m \u001b[0mi\u001b[0mn\u001b[0m \u001b[0ma\u001b[0m \u001b[0mt\u001b[0mr\u001b[0me\u001b[0ma\u001b[0ms\u001b[0mo\u001b[0mn\u001b[0mo\u001b[0mu\u001b[0ms\u001b[0m \u001b[0mt\u001b[0mr\u001b[0ma\u001b[0mi\u001b[0mt\u001b[0mo\u001b[0mr\u001b[0m.\u001b[0m\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "indices_evaluation = []\n",
+    "for i,text in enumerate(evaluation['text']):\n",
+    "    tagged_sentence = tagger_LSTM(text)   \n",
+    "    prediction_index = get_index_toxic_words(text.lower(), tagged_sentence)\n",
+    "    indices_evaluation.append(prediction_index)\n",
+    "    \n",
+    "    if i < 5:\n",
+    "        print(str(i) + colored(' Pred: ', color='cyan', attrs=['bold']) + \n",
+    "              color_toxic_words(prediction_index, text) + '\\n') "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>spans</th>\n",
+       "      <th>text</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>[118, 119, 120, 121, 122, 123, 124, 125, 126, ...</td>\n",
+       "      <td>That's right. They are not normal. And I am st...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>[81, 82, 83, 84, 85, 86]</td>\n",
+       "      <td>\"Watch people die from taking away their healt...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>[483, 484, 485, 486, 487, 488, 489, 490]</td>\n",
+       "      <td>tens years ago i contacted the PDR and suggest...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>[413, 414, 415, 416, 417, 418, 419, 420]</td>\n",
+       "      <td>The parallels between the ANC and the Sicilian...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>[150, 151, 152, 153, 154, 155, 156, 271, 272, ...</td>\n",
+       "      <td>Intel Community: ‘How can we work for a Presid...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                               spans  \\\n",
+       "0  [118, 119, 120, 121, 122, 123, 124, 125, 126, ...   \n",
+       "1                           [81, 82, 83, 84, 85, 86]   \n",
+       "2           [483, 484, 485, 486, 487, 488, 489, 490]   \n",
+       "3           [413, 414, 415, 416, 417, 418, 419, 420]   \n",
+       "4  [150, 151, 152, 153, 154, 155, 156, 271, 272, ...   \n",
+       "\n",
+       "                                                text  \n",
+       "0  That's right. They are not normal. And I am st...  \n",
+       "1  \"Watch people die from taking away their healt...  \n",
+       "2  tens years ago i contacted the PDR and suggest...  \n",
+       "3  The parallels between the ANC and the Sicilian...  \n",
+       "4  Intel Community: ‘How can we work for a Presid...  "
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "evaluation['spans'] = indices_evaluation\n",
+    "evaluation = evaluation[['spans', 'text']]\n",
+    "evaluation.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Para la evaluación se debe subir un zip con un archivo txt de la siguiente manera (al final subir el archivo `spans-pred.zip` que se produce):"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "  adding: spans-pred.txt (deflated 84%)\n"
+     ]
+    }
+   ],
+   "source": [
+    "predictions = evaluation['spans'].tolist()\n",
+    "ids = evaluation.index.tolist()\n",
+    "\n",
+    "with open(\"spans-pred.txt\", \"w\") as out:\n",
+    "    for uid, text_scores in zip(ids, predictions):\n",
+    "        out.write(f\"{str(uid)}\\t{str(text_scores)}\\n\")\n",
+    "        \n",
+    "# Zip the predictions\n",
+    "! zip -r spans-pred.zip ./spans-pred.* \n",
+    "! rm spans-pred.txt\n",
+    "! mv spans-pred.zip ../spans-pred.zip"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.3"
+  },
+  "toc-autonumbering": false
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

notebooks/utils/basic_models.py

@@ -0,0 +1,58 @@
+from nltk import pos_tag
+from nltk.tokenize import word_tokenize
+
+# The following is for the CRF, which we don't use anymore, instead we focused in the LSTM
+
+def word2features(sent, i):
+    word = sent[i][0]
+    postag = sent[i][1]
+    features = [
+        'bias',
+        'word.lower=' + word.lower(),
+        'word[-3:]=' + word[-3:],
+        'word[-2:]=' + word[-2:],
+        'word.isupper=%s' % word.isupper(),
+        'word.istitle=%s' % word.istitle(),
+        'word.isdigit=%s' % word.isdigit(),
+        'postag=' + postag,
+        'postag[:2]=' + postag[:2],
+    ]
+    if i > 0:
+        word1 = sent[i-1][0]
+        postag1 = sent[i-1][1]
+        features.extend([
+            '-1:word.lower=' + word1.lower(),
+            '-1:word.istitle=%s' % word1.istitle(),
+            '-1:word.isupper=%s' % word1.isupper(),
+            '-1:postag=' + postag1,
+            '-1:postag[:2]=' + postag1[:2],
+        ])
+    else:
+        features.append('BOS')
+        
+    if i < len(sent)-1:
+        word1 = sent[i+1][0]
+        postag1 = sent[i+1][1]
+        features.extend([
+            '+1:word.lower=' + word1.lower(),
+            '+1:word.istitle=%s' % word1.istitle(),
+            '+1:word.isupper=%s' % word1.isupper(),
+            '+1:postag=' + postag1,
+            '+1:postag[:2]=' + postag1[:2],
+        ])
+    else:
+        features.append('EOS')
+                
+    return features
+
+def sent2features(sent):
+    return [word2features(sent, i) for i in range(len(sent))]
+
+def sent2labels(sent):
+    return [label for token, postag, label in sent]
+
+def sent2tokens(sent):
+    return [token for token, postag, label in sent]
+
+def token_postag_label(sentence):
+    return pos_tag(word_tokenize(sentence))

notebooks/utils/lstm.py

@@ -0,0 +1,235 @@
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+import torch
+import torch.nn as nn
+import torch.optim as optim
+
+from torchtext.data import Field
+
+from .processing import separate_words, f1_scores
+
+import spacy
+import ast
+
+from tqdm import tqdm
+from IPython.display import clear_output
+
+nlp = spacy.load('en_core_web_md')
+dev = 'cuda:0' if torch.cuda.is_available() else 'cpu'
+torch.manual_seed(42)
+torch.backends.cudnn.deterministic = True
+
+# SpaCy hace cosas no deseadas con algunas palabras al tokenizar, como don't -> [do, n't], pero se puede corregir.
+# Pero de acuerdo a SpaCy esa es la convención, además, eso se debería codificar en los embeddings, así que se quede
+# así, sólo hay que usar el mismo tokenizador en Field de torchtext (permite el de SpaCy entre otros).
+
+# from spacy.symbols import ORTH, LEMMA, POS
+# nlp.tokenizer.add_special_case("don't", [{ORTH: "do"}, {ORTH: "not"}])
+# nlp.tokenizer.add_special_case("don't", [{ORTH: "don't"}])
+# nlp.tokenizer.add_special_case("doesn't", [{ORTH: "does"}, {ORTH: "not"}])
+
+def spacy_tokenizer	(text):
+    return [str(token) for token in nlp(text)]
+
+def prepare_data(spans, texts):
+    data = []
+    for index, text in tqdm(zip(spans, texts), total=len(texts)):
+        toxic_words = [text[i[0]:i[-1]+1] for i in separate_words(index) if len(index) > 0]
+        
+        tokens = spacy_tokenizer(text)
+        tagged_tokens = []
+        
+        for token in tokens:
+            if token in toxic_words:
+                tagged_tokens.append('toxic')
+                # Removemos en caso de que se repita posteriormente pero esté como 'non_toxic'
+                toxic_words.remove(token) 
+            else:
+                tagged_tokens.append('non_toxic')
+                
+        data.append((tokens, tagged_tokens, text, index))
+
+    return data
+
+def get_vocab(train_df):
+    train_df['text'] = train_df['text'].apply(lambda x:x.lower())
+
+    # Aquí había un problema, estábamos usando 2 tokenizadores diferentes para sacar los
+    # embeddings y para preprocesar el texto para entrenar. Pondré el de SpaCy como 
+    # tokenizador en común con el corpus de 'en_core_web_md'
+    text_field = Field(
+        tokenize='spacy',
+        tokenizer_language='en_core_web_md',
+        lower=True
+    )
+    # sadly have to apply preprocess manually
+    preprocessed_text = train_df['text'].apply(lambda x: text_field.preprocess(x))
+    # load fastext simple embedding with 200d
+    text_field.build_vocab(
+        preprocessed_text, 
+        vectors='glove.twitter.27B.200d'
+    )
+    # get the vocab instance
+    vocab = text_field.vocab
+
+    return vocab
+
+def plot_loss_and_score(train_loss, test_loss, f1_scores_train, f1_scores_test, show=True):
+    _, (ax0, ax1) = plt.subplots(nrows=1, ncols=2, figsize=(18,7))
+
+    ax0.plot(np.arange(1, len(train_loss) + 1), train_loss, marker='o', label='Train loss')
+    ax0.plot(np.arange(1, len(test_loss) + 1), test_loss, marker='o', label='Test loss')
+    ax0.set_xlabel(r'\textbf{Epochs}',size=16)
+    ax0.set_ylabel(r'\textbf{Loss}', size=16)
+    ax0.tick_params(labelsize=14)
+    ax0.legend(fontsize=14)
+
+    ax1.plot(np.arange(1, len(f1_scores_train) + 1), f1_scores_train, 
+             marker='o', label='F1 score in train')
+    ax1.plot(np.arange(1, len(f1_scores_test) + 1), f1_scores_test, 
+             marker='o', label='F1 score in test')
+    ax1.set_xlabel(r'\textbf{Epochs}',size=16)
+    ax1.set_ylabel(r'\textbf{F1 score}', size=16)
+    ax1.tick_params(labelsize=14)
+    ax1.legend(fontsize=14)
+    
+    title = 'train-F1: {:.4f} \n test-F1: {:.4f}'.format(np.max(f1_scores_train), np.max(f1_scores_test))
+    ax1.set_title(title, fontweight='bold', size=16)
+
+
+    if show:
+        plt.show()
+
+# WTF Mario, this is a mess
+def train_model(model, trainloader, testloader, stop_after_best, savefile):
+    criterion = nn.BCELoss()
+    optimizer = optim.Adam(model.parameters())
+
+    loss_per_epoch = [0]
+    training_loss = [0]
+    f1_scores_train = [0]
+    f1_scores_dev = [0]
+    best_l = None
+    best_tl = None
+    worst_l = None
+    worst_tl = None
+    worst_l_f1 = None
+    best_l_f1 = None
+    worst_tl_f1 = None
+    last_epoch_save = 0
+
+    epochs_without_change = 0
+    epochs = len(loss_per_epoch)
+
+    while epochs_without_change < stop_after_best:
+        clear_output(wait=True)
+
+        print("Training on: " + torch.cuda.get_device_name(torch.cuda.current_device()))
+        print("###############################################")
+        print("Current epoch: " + str(epochs))
+        print("Last model save was in epoch " + str(last_epoch_save))
+        print("Stopping training in: " + str(stop_after_best - epochs_without_change) + " epochs.")
+        print("###############################################")
+        print("[Best iter] training F1 is: " + str(best_tl))
+        print("[Best iter] dev F1 is: " + str(best_l))
+        print("###############################################")
+        print("[Last iter] training F1 was: " + str(f1_scores_train[-1]))
+        print("[Last iter] dev. F1 was: " + str(f1_scores_dev[-1]))
+        print("###############################################")
+        
+        # Dibujo lo que puedo
+        plot_loss_and_score(training_loss, loss_per_epoch, f1_scores_train, f1_scores_dev, show=True)
+        
+        tl = 0
+        t_pred_l = []
+        t_true_index_l = []
+        t_tokenized_l = []
+        t_text_l = []
+
+        for _, v in tqdm(enumerate(trainloader), total=len(trainloader)): # Not using batches yet
+            text = torch.reshape(v['text'], (-1,))
+            tags = torch.reshape(v['spans'], (-1,))
+            optimizer.zero_grad()
+            tag_scores = model(text)
+            
+            # Para la F1
+            t_pred_l.append(tag_scores.cpu().detach().numpy())
+            t_true_index_l.append([a.cpu().detach().numpy()[0] for a in v['true_index']])
+            t_tokenized_l.append([a[0] for a in v['tokenized']])
+            t_text_l.append(v['original_text'][0])
+            
+            loss = criterion(torch.reshape(tag_scores, (-1,)), torch.reshape(tags, (-1,)).float())
+            tl += loss.item()
+            loss.backward()
+            optimizer.step()
+
+        tl /= len(trainloader)
+        l = 0
+        print("Starting evaluation for loss function.")
+        # evaluar el modelo
+        pred_l = []
+        true_index_l = []
+        tokenized_l = []
+        text_l = []
+        
+        model.eval()
+        with torch.no_grad():
+            for v in testloader:
+                text = torch.reshape(v['text'], (-1,))
+                tags = torch.reshape(v['spans'], (-1,))
+
+                tag_scores = model(text)
+                
+                #Para la F1
+                pred_l.append(tag_scores.cpu().detach().numpy())
+                true_index_l.append([a.cpu().detach().numpy()[0] for a in v['true_index']])
+                tokenized_l.append([a[0] for a in v['tokenized']])
+                text_l.append(v['original_text'][0])
+                
+                loss = criterion(torch.reshape(tag_scores, (-1,)), torch.reshape(tags, (-1,)).float())
+                l += loss.item()
+        
+        model.train()
+        l /= len(testloader)
+        print("Starting evaluation for dev F1")
+        f1_d = f1_scores(pred_l, true_index_l, tokenized_l, text_l)
+        # Es aproximado, pero solo es una referencia
+        f1_t = f1_scores(t_pred_l, t_true_index_l, t_tokenized_l, t_text_l) 
+        
+        
+        epochs_without_change += 1
+        if best_l is None or best_l < f1_d:
+            print("Model improved, saving.")
+            torch.save(model, savefile)
+            best_l = f1_d
+            best_tl = f1_t
+            epochs_without_change = 0
+            last_epoch_save = epochs
+            print("Model improved, saved.")
+
+        # Para graficar con una escala coherente.
+        if(worst_l_f1 is None or f1_d < worst_l_f1):
+            worst_l_f1 = f1_d
+            f1_scores_dev[0] = worst_l_f1
+        if(worst_tl_f1 is None or f1_t < worst_tl_f1):
+            worst_tl_f1 = f1_t
+            f1_scores_train[0] = worst_tl_f1
+        if(worst_tl is None or tl > worst_tl):
+            worst_tl = tl
+            training_loss[0] = worst_tl
+        if(worst_l is None or l > worst_l):
+            worst_l = l
+            loss_per_epoch[0] = worst_l
+
+        # Rastreo las perdidas
+        loss_per_epoch.append(l)
+        training_loss.append(tl)
+        f1_scores_train.append(f1_t)
+        f1_scores_dev.append(f1_d)
+        # Rastreo la época actual
+        epochs += 1
+    print('Finished Training')
+
+    return loss_per_epoch, training_loss, f1_scores_train, f1_scores_dev

notebooks/utils/processing.py

@@ -0,0 +1,129 @@
+from termcolor import colored
+import string
+
+def color_toxic_words(index, text, html=False):
+    if not html:
+        colored_string = ''
+        for i, x in enumerate(text):
+            if i in index:
+                colored_string += colored(x, on_color='on_red')
+            else:
+                colored_string += colored(x)
+    else:
+        colored_string = ''
+        for i, x in enumerate(text):
+            if i in index:
+                colored_string += f'<span style="background-color: #FF0000">{x}</span>'
+            else:
+                colored_string += x
+            
+    return colored_string
+
+def remove_symbols(index, text):
+    """
+    Remueve los índices que corresponden a símbolos 'no tóxicos', como espacios en blanco
+    comas, puntos, etc.
+    """
+    index_clean = []   
+    for i in index:
+        x = text[i]
+        if x not in ('"()+,-./:;<=>[\\]^_`{|}~' + string.whitespace):
+            index_clean.append(i)
+                        
+    return index_clean
+
+def completely_toxic(span, text):
+    if span == []:
+        return [i for i in range(len(text))]
+    else:
+    	return span
+
+def separate_words(indices):
+    """
+    Separa los índices por palabras.
+    """
+
+    toxic_words_indices = []
+    m = 0
+    for i,(j,k) in enumerate(zip(indices[0:-1], indices[1:])):
+        if k-j != 1:
+            toxic_words_indices.append(indices[m:i+1])
+            m = i+1
+    toxic_words_indices.append(indices[m:]) # Última palabra
+    
+    return toxic_words_indices   
+
+
+def postprocessing(indices_list, delta=7):
+    """
+    Pone como tóxicos los caracteres en medio de dos palabras tóxicas si el espacio
+    entre ellas es menor a delta.
+    """
+
+    # Asumiendo que tienes indices numéricos enteros.
+    if len(indices_list) > 1:
+        l = sorted(indices_list)
+        new_list = []
+        for i in range(len(indices_list)-1):
+            # Agrego el indice existente
+            new_list.append(l[i])
+            # Si no hay mucho espacio entre este y el siguiente indice, selecciono todos los indices intermedios
+            if (l[i+1] - l[i]) <= delta:
+                new_list = new_list + list(range(l[i]+1,l[i+1]))
+                
+        new_list.append(l[-1]) # El ultimo elemento
+        return new_list
+    else:
+        return indices_list
+
+
+def get_index_toxic_words(sentence, tagged_sentence, delta=7):
+    toxic_indices = []   
+    m = 0
+    #tag_to_ix = {"non_toxic": 0, "toxic": 1}
+    for word_tag in tagged_sentence:
+        word, tag = word_tag    
+        if tag == 1: #toxic
+            # Si la palabra tóxica aparece 2 o más veces ésto solo dará la primera 
+            # aparición, hay que arreglar eso pero por lo mientras sirve
+            # word_indices = [sentence.find(word) + i for i in range(len(word))]
+            # toxic_indices.append(word_indices)
+            
+            # Así parece evitar el problema de la palabra repetida
+            word_indices = [m + sentence[m:].find(word) + i for i in range(len(word))]
+            toxic_indices.append(word_indices)
+        # Ya se arregla el 'bug' de 'stupidity'
+        m += sentence[m:].find(word) + len(word)
+            
+    toxic_indices = [val for sublist in toxic_indices for val in sublist]
+    
+    # Unir espacios y otras cosas para que suba el F1    
+    return postprocessing(toxic_indices, delta)
+
+
+def f1(predictions, gold):
+    """
+    F1 (a.k.a. DICE) operating on two lists of offsets (e.g., character).
+    >>> assert f1([0, 1, 4, 5], [0, 1, 6]) == 0.5714285714285714
+    :param predictions: a list of predicted offsets
+    :param gold: a list of offsets serving as the ground truth
+    :return: a score between 0 and 1
+    """
+    if len(gold) == 0:
+        return 1. if len(predictions) == 0 else 0.
+    if len(predictions) == 0:
+        return 0.
+    predictions_set = set(predictions)
+    gold_set = set(gold)
+    nom = 2 * len(predictions_set.intersection(gold_set))
+    denom = len(predictions_set) + len(gold_set)
+    return float(nom)/float(denom)
+
+def f1_scores(pred, true_index, tokenized, text, threshold=0.5):
+    scores = 0
+    for i in range(len(pred)):
+        tags = [1 if x > threshold else 0 for x in pred[i]]
+        tagged_sentence = list(zip(tokenized[i], tags))
+        prediction_index = get_index_toxic_words(text[i], tagged_sentence)
+        scores += f1(prediction_index, true_index[i])
+    return scores/len(pred)