Updated the notebook

flisol-cordoba-2023.ipynb

@@ -258,7 +258,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "0e0d53be",
    "metadata": {
     "slideshow": {
@@ -276,7 +276,9 @@
     "\n",
     "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
     "tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)\n",
-    "model = AutoModelForCausalLM.from_pretrained(BASE_MODEL, low_cpu_mem_usage=True, torch_dtype=\"auto\").to(device)"
+    "model = AutoModelForCausalLM.from_pretrained(\n",
+    "    BASE_MODEL, torch_dtype=\"auto\"\n",
+    ").to(device)"
    ]
   },
   {
@@ -339,7 +341,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "c1227c49",
    "metadata": {
     "slideshow": {
@@ -378,32 +380,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "id": "11bec6de",
    "metadata": {
     "slideshow": {
      "slide_type": "fragment"
     }
    },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "La siguiente es una conversación entre un HUMANO y un bot EXPERTO en software libre.\n",
-      "El EXPERTO le ayuda al HUMANO con preguntas acerca de software libre.\n",
-      "El EXPERTO es conversacional, optimista, flexible, creativo y genera respuestas parecidas a un humano.\n",
-      "\n",
-      "HUMANO: Hola, ¿Cómo estás?\n",
-      "EXPERTO: Hola, pmuy bien. Estoy acá para ayudarte con preguntas respecto al software libre.\n",
-      "\n",
-      "HUMANO: ¿Qué es el software libre?\n",
-      "EXPERTO: El software libre es un software que se puede modificar, redistribuir y distribuir libremente.\n",
-      "HUMANO: ¿En qué consiste la licencia GPL?\n",
-      "EXPERTO: La licencia GPL es una licencia de software libre que permite a los usuarios modificar, redistribuir\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "MAX_TOKENS = 50\n",
     "input_ids = tokenizer.encode(PROMPT, return_tensors=\"pt\").to(device)\n",
@@ -433,33 +417,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "id": "dc66f288",
    "metadata": {
     "slideshow": {
      "slide_type": "fragment"
     }
    },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "La siguiente es una conversación entre un HUMANO y un bot EXPERTO en software libre.\n",
-      "El EXPERTO le ayuda al HUMANO con preguntas acerca de software libre.\n",
-      "El EXPERTO es conversacional, optimista, flexible, creativo y genera respuestas parecidas a un humano.\n",
-      "\n",
-      "HUMANO: Hola, ¿Cómo estás?\n",
-      "EXPERTO: Hola, pmuy bien. Estoy acá para ayudarte con preguntas respecto al software libre.\n",
-      "\n",
-      "HUMANO: ¿Qué es el software libre?\n",
-      "EXPERTO: El software libre, es aquel software que esta escrito en un lenguaje de programación que puede ser modificado y copiado por cualquier persona o entidad.\n",
-      "\n",
-      "HUMANO: ¿En general cuáles son los usos que se pueden dar a un software libre?\n",
-      "EXPERTO\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "torch.manual_seed(42)  # To ensure determinism\n",
     "\n",
@@ -604,7 +569,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "17f2884d",
    "metadata": {
     "slideshow": {
@@ -616,6 +581,8 @@
     "import torch\n",
     "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
     "\n",
+    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+    "\n",
     "BASE_MODEL = \"DeepESP/gpt2-spanish\"  # We play with a smaller model\n",
     "tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)\n",
     "model = AutoModelForCausalLM.from_pretrained(BASE_MODEL).to(device)"
@@ -637,26 +604,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "322a4a9b",
    "metadata": {
     "slideshow": {
      "slide_type": "fragment"
     }
    },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Aquí me pongo a cantar y a llorar. \n",
-      "\n",
-      "Los sollozos de Meggie se desvanecen por la noche en el salón. Al parecer no se ve nada. \n",
-      "\n",
-      "—¿Y si no fuera el final del mundo, el fin de un mundo?\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "torch.manual_seed(42)  # To ensure determinism\n",
     "\n",
@@ -686,30 +641,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "id": "5a27197e",
    "metadata": {
     "slideshow": {
      "slide_type": "fragment"
     }
    },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "I - Cantor y Gaucho.\n",
-      "\n",
-      "1\n",
-      "Aquí me pongo a cantar\n",
-      "Al compás de la vigüela,\n",
-      "Que el hombre que lo desvela\n",
-      "Una pena estraordinaria\n",
-      "Como la ave solitaria\n",
-      "Con el cantar se consuela.\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "from datasets import load_dataset\n",
     "\n",
@@ -740,7 +679,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "id": "33059c5f",
    "metadata": {
     "scrolled": true,
@@ -778,7 +717,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "id": "3100e195",
    "metadata": {
     "scrolled": true,
@@ -817,23 +756,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "id": "b9d33b7b",
    "metadata": {
     "slideshow": {
      "slide_type": "fragment"
     }
    },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "128\n",
-      "[50, 1368, 6505, 282, 324, 24275, 526, 23, 208, 208]\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "print(len(lm_datasets[\"train\"][0][\"input_ids\"]))\n",
     "print(lm_datasets[\"train\"][0][\"input_ids\"][:10])"
@@ -841,44 +771,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "id": "7dfb316d",
    "metadata": {
     "slideshow": {
      "slide_type": "fragment"
     }
    },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "I - Cantor y Gaucho.\n",
-      "\n",
-      "1\n",
-      "Aquí me pongo a cantar\n",
-      "Al compás de la vigüela,\n",
-      "Que el hombre que lo desvela\n",
-      "Una pena estraordinaria\n",
-      "Como la ave solitaria\n",
-      "Con el cantar se consuela.\n",
-      "\n",
-      "2\n",
-      "Pido a los Santos del Cielo\n",
-      "Que ayuden mi pensamiento;\n",
-      "Les pido en este momento\n",
-      "Que voy a cantar mi historia\n",
-      "Me refresquen la memoria\n",
-      "Y aclaren mi entendimiento.\n",
-      "\n",
-      "3\n",
-      "Vengan Santos milagrosos,\n",
-      "Vengan todos en mi ayuda,\n",
-      "Que la lengua se me añuda\n",
-      "Y se me turba\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "print(tokenizer.decode(lm_datasets[\"train\"][0][\"input_ids\"]))"
    ]
@@ -900,7 +800,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "id": "a8b90ba2",
    "metadata": {
     "scrolled": true,
@@ -935,100 +835,21 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
-   "id": "3b121d21",
+   "execution_count": null,
+   "id": "ccd8e608-7e14-4796-9e52-c55b6df3ce6f",
    "metadata": {
     "slideshow": {
      "slide_type": "subslide"
     }
    },
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "\n",
-       "    <div>\n",
-       "      \n",
-       "      <progress value='180' max='180' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
-       "      [180/180 11:44, Epoch 10/10]\n",
-       "    </div>\n",
-       "    <table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       " <tr style=\"text-align: left;\">\n",
-       "      <th>Epoch</th>\n",
-       "      <th>Training Loss</th>\n",
-       "      <th>Validation Loss</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <td>1</td>\n",
-       "      <td>4.386400</td>\n",
-       "      <td>4.202457</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <td>2</td>\n",
-       "      <td>3.948000</td>\n",
-       "      <td>4.043974</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <td>3</td>\n",
-       "      <td>3.796200</td>\n",
-       "      <td>3.980350</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <td>4</td>\n",
-       "      <td>3.610500</td>\n",
-       "      <td>3.945783</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <td>5</td>\n",
-       "      <td>3.444400</td>\n",
-       "      <td>3.927984</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <td>6</td>\n",
-       "      <td>3.385500</td>\n",
-       "      <td>3.919229</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <td>7</td>\n",
-       "      <td>3.314200</td>\n",
-       "      <td>3.909090</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <td>8</td>\n",
-       "      <td>3.219200</td>\n",
-       "      <td>3.907399</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <td>9</td>\n",
-       "      <td>3.161500</td>\n",
-       "      <td>3.906959</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <td>10</td>\n",
-       "      <td>3.163700</td>\n",
-       "      <td>3.906726</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table><p>"
-      ],
-      "text/plain": [
-       "<IPython.core.display.HTML object>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
     "from transformers import Trainer, TrainingArguments\n",
     "\n",
     "training_args = TrainingArguments(\n",
     "    \"flisol-cba-martin-fierro\",\n",
-    "    evaluation_strategy=\"epoch\",\n",
-    "    num_train_epochs=10,\n",
+    "    eval_strategy=\"epoch\",\n",
+    "    num_train_epochs=25,\n",
     "    learning_rate=2e-5,\n",
     "    weight_decay=0.01,\n",
     "    logging_steps=5,\n",
@@ -1041,7 +862,20 @@
     "    eval_dataset=lm_datasets[\"validation\"],\n",
     ")\n",
     "\n",
-    "trainer.train()\n",
+    "trainer.train();"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "16695fc7-9baa-41f7-9367-d8fbcd987b79",
+   "metadata": {
+    "slideshow": {
+     "slide_type": "subslide"
+    }
+   },
+   "outputs": [],
+   "source": [
     "trainer.push_to_hub()  # This pushes the trained model to Hugging Face model repository\n",
     "tokenizer.push_to_hub(\"flisol-cba-martin-fierro\")"
    ]
@@ -1064,30 +898,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "6a35e80f",
    "metadata": {
     "slideshow": {
      "slide_type": "fragment"
     }
    },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Aquí me pongo a cantar;\n",
-      "y si tengo el sueño:\n",
-      "de pronto se me ha quedado la sangre:\n",
-      "como te asombre se me\n",
-      "lo oí decir muchas veces,\n",
-      "pero el tiempo me ha borrado.\n",
-      "\n",
-      "2\n",
-      "Soy\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "import torch\n",
     "from transformers import AutoModelForCausalLM, AutoTokenizer\n",