Finished presentation notebook

flisol-cordoba-2023.ipynb

@@ -261,7 +261,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "0e0d53be",
    "metadata": {
     "slideshow": {
@@ -330,7 +330,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "c1227c49",
    "metadata": {
     "slideshow": {
@@ -369,14 +369,32 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "11bec6de",
    "metadata": {
     "slideshow": {
      "slide_type": "fragment"
     }
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "La siguiente es una conversación entre un HUMANO y un bot EXPERTO en software libre.\n",
+      "El EXPERTO le ayuda al HUMANO con preguntas acerca de software libre.\n",
+      "El EXPERTO es conversacional, optimista, flexible, creativo y genera respuestas parecidas a un humano.\n",
+      "\n",
+      "HUMANO: Hola, ¿Cómo estás?\n",
+      "EXPERTO: Hola, pmuy bien. Estoy acá para ayudarte con preguntas respecto al software libre.\n",
+      "\n",
+      "HUMANO: ¿Qué es el software libre?\n",
+      "EXPERTO: El software libre es un software que se puede modificar, redistribuir y distribuir libremente.\n",
+      "HUMANO: ¿En qué consiste la licencia GPL?\n",
+      "EXPERTO: La licencia GPL es una licencia de software libre que permite a los usuarios modificar, redistribuir\n"
+     ]
+    }
+   ],
    "source": [
     "MAX_TOKENS = 50\n",
     "input_ids = tokenizer.encode(PROMPT, return_tensors='pt')\n",
@@ -406,14 +424,33 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "id": "dc66f288",
    "metadata": {
     "slideshow": {
      "slide_type": "fragment"
     }
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "La siguiente es una conversación entre un HUMANO y un bot EXPERTO en software libre.\n",
+      "El EXPERTO le ayuda al HUMANO con preguntas acerca de software libre.\n",
+      "El EXPERTO es conversacional, optimista, flexible, creativo y genera respuestas parecidas a un humano.\n",
+      "\n",
+      "HUMANO: Hola, ¿Cómo estás?\n",
+      "EXPERTO: Hola, pmuy bien. Estoy acá para ayudarte con preguntas respecto al software libre.\n",
+      "\n",
+      "HUMANO: ¿Qué es el software libre?\n",
+      "EXPERTO: El software libre, es aquel software que esta escrito en un lenguaje de programación que puede ser modificado y copiado por cualquier persona o entidad.\n",
+      "\n",
+      "HUMANO: ¿En general cuáles son los usos que se pueden dar a un software libre?\n",
+      "EXPERTO\n"
+     ]
+    }
+   ],
    "source": [
     "torch.manual_seed(42)  # To ensure determinism\n",
     "\n",
@@ -473,8 +510,7 @@
     "\"\"\".strip()\n",
     "\n",
     "chatbot = ChatBot(\n",
-    "    base_model=model,\n",
-    "    tokenizer=tokenizer,\n",
+    "    base_model='bigscience/bloom-3b',\n",
     "    initial_prompt=PROMPT,\n",
     "    keep_context=True,\n",
     "    creative=True,\n",
@@ -597,14 +633,6 @@
     }
    },
    "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
-      "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
-     ]
-    },
     {
      "name": "stdout",
      "output_type": "stream",
@@ -654,17 +682,10 @@
     }
    },
    "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Found cached dataset text (/home/crscardellino/.cache/huggingface/datasets/text/default-623d9572e8f69157/0.0.0/cb1e9bd71a82ad27976be3b12b407850fe2837d80c22c5e03a28949843a8ace2)\n"
-     ]
-    },
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "0fe0bb8953f24e05b2a56ad08c462976",
+       "model_id": "123690f207a94d3e850acef7a13133a6",
        "version_major": 2,
        "version_minor": 0
       },
@@ -724,16 +745,7 @@
      "slide_type": "fragment"
     }
    },
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Loading cached processed dataset at /home/crscardellino/.cache/huggingface/datasets/text/default-623d9572e8f69157/0.0.0/cb1e9bd71a82ad27976be3b12b407850fe2837d80c22c5e03a28949843a8ace2/cache-5a0f77d99160fc1c_*_of_00004.arrow\n",
-      "Loading cached processed dataset at /home/crscardellino/.cache/huggingface/datasets/text/default-623d9572e8f69157/0.0.0/cb1e9bd71a82ad27976be3b12b407850fe2837d80c22c5e03a28949843a8ace2/cache-003d85e2eebe3231_*_of_00004.arrow\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "from utils import tokenize  # local module in the repository\n",
     "\n",
@@ -769,16 +781,7 @@
      "slide_type": "fragment"
     }
    },
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Loading cached processed dataset at /home/crscardellino/.cache/huggingface/datasets/text/default-623d9572e8f69157/0.0.0/cb1e9bd71a82ad27976be3b12b407850fe2837d80c22c5e03a28949843a8ace2/cache-01936c1905752293_*_of_00004.arrow\n",
-      "Loading cached processed dataset at /home/crscardellino/.cache/huggingface/datasets/text/default-623d9572e8f69157/0.0.0/cb1e9bd71a82ad27976be3b12b407850fe2837d80c22c5e03a28949843a8ace2/cache-af8dcd60a546c28d_*_of_00004.arrow\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "from functools import partial\n",
     "from utils import group_texts  # local module in the repository\n",
@@ -903,13 +906,18 @@
    },
    "outputs": [
     {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Token is valid.\n",
-      "Your token has been saved to /home/crscardellino/.cache/huggingface/token\n",
-      "Login successful\n"
-     ]
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "94b41ffd721d4bbf8840df3fee46bbb2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
     }
    ],
    "source": [
@@ -946,14 +954,6 @@
     }
    },
    "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/home/crscardellino/Projects/research/flisol/flisol-cba-martin-fierro/venv/lib/python3.10/site-packages/transformers/optimization.py:391: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
-      "  warnings.warn(\n"
-     ]
-    },
     {
      "data": {
       "text/html": [
@@ -1031,16 +1031,6 @@
      },
      "metadata": {},
      "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/plain": [
-       "TrainOutput(global_step=180, training_loss=3.5808190133836533, metrics={'train_runtime': 707.4357, 'train_samples_per_second': 1.951, 'train_steps_per_second': 0.254, 'total_flos': 90145751040000.0, 'train_loss': 3.5808190133836533, 'epoch': 10.0})"
-      ]
-     },
-     "execution_count": 9,
-     "metadata": {},
-     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -1062,20 +1052,7 @@
     "    eval_dataset=lm_datasets[\"validation\"]\n",
     ")\n",
     "\n",
-    "trainer.train()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "d43c5555",
-   "metadata": {
-    "slideshow": {
-     "slide_type": "-"
-    }
-   },
-   "outputs": [],
-   "source": [
+    "trainer.train()\n",
     "trainer.push_to_hub()  # This pushes the trained model to Hugging Face model repository"
    ]
   },
@@ -1097,21 +1074,36 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "6a35e80f",
    "metadata": {
     "slideshow": {
      "slide_type": "fragment"
     }
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Aquí me pongo a cantar;\n",
+      "y si tengo el sueño:\n",
+      "de pronto se me ha quedado la sangre:\n",
+      "como te asombre se me\n",
+      "lo oí decir muchas veces,\n",
+      "pero el tiempo me ha borrado.\n",
+      "\n",
+      "2\n",
+      "Soy\n"
+     ]
+    }
+   ],
    "source": [
     "import torch\n",
     "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
     "\n",
-    "BASE_MODEL = \"crscardellino/flisol-cba-martin-fierro\"\n",
-    "tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)\n",
-    "model = AutoModelForCausalLM.from_pretrained(BASE_MODEL)\n",
+    "tokenizer = AutoTokenizer.from_pretrained(\"DeepESP/gpt2-spanish\")\n",
+    "model = AutoModelForCausalLM.from_pretrained(\"crscardellino/flisol-cba-martin-fierro\")\n",
     "\n",
     "torch.manual_seed(42)  # To ensure determinism\n",
     "\n",